port-ocean 0.24.22__py3-none-any.whl → 0.25.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- integrations/_infra/Dockerfile.Deb +5 -2
- integrations/_infra/Dockerfile.local +10 -3
- integrations/_infra/README.md +30 -0
- integrations/_infra/entry_local.sh +2 -1
- port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py +7 -0
- port_ocean/core/handlers/entity_processor/jq_entity_processor.py +62 -21
- port_ocean/core/integrations/mixins/sync_raw.py +307 -117
- port_ocean/core/ocean_types.py +1 -1
- port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py +22 -13
- port_ocean/tests/core/handlers/mixins/test_live_events.py +2 -2
- port_ocean/tests/core/handlers/mixins/test_sync_raw.py +8 -8
- {port_ocean-0.24.22.dist-info → port_ocean-0.25.2.dist-info}/METADATA +1 -1
- {port_ocean-0.24.22.dist-info → port_ocean-0.25.2.dist-info}/RECORD +16 -15
- {port_ocean-0.24.22.dist-info → port_ocean-0.25.2.dist-info}/LICENSE.md +0 -0
- {port_ocean-0.24.22.dist-info → port_ocean-0.25.2.dist-info}/WHEEL +0 -0
- {port_ocean-0.24.22.dist-info → port_ocean-0.25.2.dist-info}/entry_points.txt +0 -0
@@ -3,6 +3,7 @@ ARG BASE_RUNNER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-runner:latest
|
|
3
3
|
|
4
4
|
FROM ${BASE_BUILDER_PYTHON_IMAGE} AS base
|
5
5
|
|
6
|
+
ARG OCEAN_USER_ID=999
|
6
7
|
ARG BUILD_CONTEXT
|
7
8
|
ARG BUILDPLATFORM
|
8
9
|
|
@@ -19,7 +20,7 @@ RUN poetry install --without dev --no-root --no-interaction --no-ansi --no-cache
|
|
19
20
|
|
20
21
|
FROM ${BASE_RUNNER_PYTHON_IMAGE} AS prod
|
21
22
|
|
22
|
-
RUN groupadd -r appgroup && useradd -r -g appgroup -m ocean
|
23
|
+
RUN groupadd -r appgroup && useradd -r -g appgroup -m -u ${OCEAN_USER_ID} ocean
|
23
24
|
|
24
25
|
RUN mkdir -p /tmp/ocean
|
25
26
|
|
@@ -38,6 +39,7 @@ RUN apt-get update \
|
|
38
39
|
ca-certificates \
|
39
40
|
openssl \
|
40
41
|
curl \
|
42
|
+
acl \
|
41
43
|
&& apt-get clean
|
42
44
|
|
43
45
|
LABEL INTEGRATION_VERSION=${INTEGRATION_VERSION}
|
@@ -64,7 +66,8 @@ RUN chmod a+x /app/.venv/bin/ocean
|
|
64
66
|
|
65
67
|
RUN chmod a+x /app/init.sh
|
66
68
|
RUN ln -s /app/.venv/bin/ocean /usr/bin/ocean
|
67
|
-
|
69
|
+
# Add ocean user to ssl certs group
|
70
|
+
RUN setfacl -m u:ocean:rwX /etc/ssl/certs
|
68
71
|
USER ocean
|
69
72
|
# Run the application
|
70
73
|
CMD ["bash", "/app/init.sh"]
|
@@ -2,9 +2,10 @@ ARG BASE_PYTHON_IMAGE=debian:trixie-slim
|
|
2
2
|
# debian:trixie-slim - Python 3.12
|
3
3
|
FROM ${BASE_PYTHON_IMAGE}
|
4
4
|
|
5
|
-
|
5
|
+
ARG OCEAN_USER_ID=999
|
6
|
+
RUN groupadd -r appgroup && useradd -r -g appgroup -m -u ${OCEAN_USER_ID} ocean
|
6
7
|
|
7
|
-
RUN mkdir -p /tmp/ocean
|
8
|
+
RUN mkdir -p /tmp/ocean
|
8
9
|
|
9
10
|
|
10
11
|
RUN apt-get update \
|
@@ -27,6 +28,7 @@ RUN apt-get update \
|
|
27
28
|
build-essential\
|
28
29
|
git \
|
29
30
|
python3-venv \
|
31
|
+
acl \
|
30
32
|
&& apt-get clean
|
31
33
|
|
32
34
|
ARG BUILD_CONTEXT
|
@@ -40,7 +42,6 @@ ENV PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
|
|
40
42
|
RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
|
41
43
|
|
42
44
|
WORKDIR /app
|
43
|
-
USER ocean
|
44
45
|
|
45
46
|
COPY . .
|
46
47
|
RUN rm -rf .venv-docker ${BUILD_CONTEXT}/.venv-docker
|
@@ -51,5 +52,11 @@ RUN python3 -m venv ${BUILD_CONTEXT}/.venv-docker
|
|
51
52
|
WORKDIR /app/${BUILD_CONTEXT}
|
52
53
|
|
53
54
|
WORKDIR /app
|
55
|
+
RUN chown -R ocean:appgroup /app && chmod -R 755 /app
|
56
|
+
RUN chown -R ocean:appgroup /app/${BUILD_CONTEXT} && chmod -R 755 /app/${BUILD_CONTEXT}
|
57
|
+
RUN chown -R ocean:appgroup /tmp/ocean && chmod -R 755 /tmp/ocean
|
58
|
+
# Add ocean user to ssl certs group
|
59
|
+
RUN setfacl -m u:ocean:rwX /etc/ssl/certs
|
60
|
+
USER ocean
|
54
61
|
|
55
62
|
ENTRYPOINT ["./integrations/_infra/entry_local.sh"]
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# Running the Ocean image locally using vscode / Cursor
|
2
|
+
|
3
|
+
In order to run the local image of Ocean you need to follow these steps:
|
4
|
+
|
5
|
+
1. Build the image:
|
6
|
+
|
7
|
+
```bash
|
8
|
+
docker build -f integrations/_infra/Dockerfile.local --build-arg BUILD_CONTEXT=integrations/<integration_type> --platform linux/arm64 -t <my-local-image>:<local> .
|
9
|
+
```
|
10
|
+
|
11
|
+
2. Run the image
|
12
|
+
1. `5678` is the debugpy port mentioned in the `entry_local.sh` file
|
13
|
+
2. `8000` is the port of the Ocean FastAPI server
|
14
|
+
3. the `-v` option mounts your local Ocean directory to the pod, allowing you not to constantly build the image.
|
15
|
+
1. Make sure to run the command from the root directory of the Ocean repository.
|
16
|
+
|
17
|
+
```bash
|
18
|
+
docker run --rm -it \
|
19
|
+
-v $(pwd):/app \
|
20
|
+
-p 5678:5678 \
|
21
|
+
-p 8000:8000 \
|
22
|
+
-e BUILD_CONTEXT=integrations/<integration_type> \
|
23
|
+
-e OCEAN__PORT__CLIENT_ID=<MY_CLIENT_ID> \
|
24
|
+
-e OCEAN__PORT__CLIENT_SECRET=<MY_CLIENT_SECRET> \
|
25
|
+
-e OCEAN__PORT__MY_OTHER_CONFIGURATION=<MY_OTHER_CONFIGURATION> \
|
26
|
+
<my-local-image>:<local>
|
27
|
+
```
|
28
|
+
|
29
|
+
3. In vscode/Cursor, run the `Attach to docker fake-integration integration` Running configuration from the `launch.json`.
|
30
|
+
4. Have fun debugging!
|
@@ -2,6 +2,8 @@ import asyncio
|
|
2
2
|
from collections import defaultdict
|
3
3
|
from itertools import groupby
|
4
4
|
|
5
|
+
from loguru import logger
|
6
|
+
|
5
7
|
from port_ocean.clients.port.client import PortClient
|
6
8
|
from port_ocean.core.models import Entity
|
7
9
|
|
@@ -37,6 +39,11 @@ async def get_related_entities(
|
|
37
39
|
blueprints_to_relations = defaultdict(list)
|
38
40
|
for entity, blueprint in entity_to_blueprint:
|
39
41
|
for relation_name, relation in entity.relations.items():
|
42
|
+
if relation_name not in blueprint.relations:
|
43
|
+
logger.warning(
|
44
|
+
f"Relation {relation_name} found in entity {entity.identifier} but not in blueprint {blueprint.identifier}"
|
45
|
+
)
|
46
|
+
continue
|
40
47
|
relation_blueprint = blueprint.relations[relation_name].target
|
41
48
|
blueprints_to_relations[relation_blueprint].extend(
|
42
49
|
relation if isinstance(relation, list) else [relation]
|
@@ -1,6 +1,7 @@
|
|
1
1
|
import asyncio
|
2
2
|
from asyncio import Task
|
3
3
|
from dataclasses import dataclass, field
|
4
|
+
|
4
5
|
from functools import lru_cache
|
5
6
|
from typing import Any, Optional
|
6
7
|
import jq # type: ignore
|
@@ -23,6 +24,49 @@ from port_ocean.exceptions.core import EntityProcessorException
|
|
23
24
|
from port_ocean.utils.queue_utils import process_in_queue
|
24
25
|
|
25
26
|
|
27
|
+
class ExampleStates:
|
28
|
+
__succeed: list[dict[str, Any]]
|
29
|
+
__errors: list[dict[str, Any]]
|
30
|
+
__max_size: int
|
31
|
+
|
32
|
+
def __init__(self, max_size: int = 0) -> None:
|
33
|
+
"""
|
34
|
+
Store two sequences:
|
35
|
+
- succeed: items that succeeded
|
36
|
+
- errors: items that failed
|
37
|
+
"""
|
38
|
+
self.__succeed = []
|
39
|
+
self.__errors = []
|
40
|
+
self.__max_size = max_size
|
41
|
+
|
42
|
+
def add_example(self, succeed: bool, item: dict[str, Any]) -> None:
|
43
|
+
if succeed:
|
44
|
+
self.__succeed.append(item)
|
45
|
+
else:
|
46
|
+
self.__errors.append(item)
|
47
|
+
|
48
|
+
def __len__(self) -> int:
|
49
|
+
"""
|
50
|
+
Total number of items (successes + errors).
|
51
|
+
"""
|
52
|
+
return len(self.__succeed) + len(self.__errors)
|
53
|
+
|
54
|
+
def get_examples(self, number: int = 0) -> list[dict[str, Any]]:
|
55
|
+
"""
|
56
|
+
Return a list of up to number items, taking successes first,
|
57
|
+
"""
|
58
|
+
if number <= 0:
|
59
|
+
number = self.__max_size
|
60
|
+
# how many from succeed?
|
61
|
+
s_count = min(number, len(self.__succeed))
|
62
|
+
result = list(self.__succeed[:s_count])
|
63
|
+
# how many more from errors?
|
64
|
+
e_count = number - s_count
|
65
|
+
if e_count > 0:
|
66
|
+
result.extend(self.__errors[:e_count])
|
67
|
+
return result
|
68
|
+
|
69
|
+
|
26
70
|
@dataclass
|
27
71
|
class MappedEntity:
|
28
72
|
"""Represents the entity after applying the mapping
|
@@ -182,11 +226,16 @@ class JQEntityProcessor(BaseEntityProcessor):
|
|
182
226
|
return MappedEntity(
|
183
227
|
mapped_entity,
|
184
228
|
did_entity_pass_selector=should_run,
|
185
|
-
raw_data=data
|
229
|
+
raw_data=data,
|
186
230
|
misconfigurations=misconfigurations,
|
187
231
|
)
|
188
232
|
|
189
|
-
return MappedEntity(
|
233
|
+
return MappedEntity(
|
234
|
+
{},
|
235
|
+
did_entity_pass_selector=False,
|
236
|
+
raw_data=data,
|
237
|
+
misconfigurations={},
|
238
|
+
)
|
190
239
|
|
191
240
|
async def _calculate_entity(
|
192
241
|
self,
|
@@ -264,24 +313,26 @@ class JQEntityProcessor(BaseEntityProcessor):
|
|
264
313
|
|
265
314
|
passed_entities = []
|
266
315
|
failed_entities = []
|
267
|
-
examples_to_send
|
316
|
+
examples_to_send = ExampleStates(send_raw_data_examples_amount)
|
268
317
|
entity_misconfigurations: dict[str, str] = {}
|
269
318
|
missing_required_fields: bool = False
|
270
319
|
entity_mapping_fault_counter: int = 0
|
271
|
-
|
272
320
|
for result in calculated_entities_results:
|
273
321
|
if len(result.misconfigurations) > 0:
|
274
322
|
entity_misconfigurations |= result.misconfigurations
|
275
323
|
|
324
|
+
if (
|
325
|
+
len(examples_to_send) < send_raw_data_examples_amount
|
326
|
+
and result.raw_data is not None
|
327
|
+
):
|
328
|
+
examples_to_send.add_example(
|
329
|
+
result.did_entity_pass_selector, result.raw_data
|
330
|
+
)
|
331
|
+
|
276
332
|
if result.entity.get("identifier") and result.entity.get("blueprint"):
|
277
333
|
parsed_entity = Entity.parse_obj(result.entity)
|
278
334
|
if result.did_entity_pass_selector:
|
279
335
|
passed_entities.append(parsed_entity)
|
280
|
-
if (
|
281
|
-
len(examples_to_send) < send_raw_data_examples_amount
|
282
|
-
and result.raw_data is not None
|
283
|
-
):
|
284
|
-
examples_to_send.append(result.raw_data)
|
285
336
|
else:
|
286
337
|
failed_entities.append(parsed_entity)
|
287
338
|
else:
|
@@ -294,20 +345,10 @@ class JQEntityProcessor(BaseEntityProcessor):
|
|
294
345
|
entity_mapping_fault_counter,
|
295
346
|
)
|
296
347
|
|
297
|
-
|
298
|
-
not calculated_entities_results
|
299
|
-
and raw_results
|
300
|
-
and send_raw_data_examples_amount > 0
|
301
|
-
):
|
302
|
-
logger.warning(
|
303
|
-
f"No entities were parsed from {len(raw_results)} raw results, sending raw data examples"
|
304
|
-
)
|
305
|
-
examples_to_send = raw_results[:send_raw_data_examples_amount]
|
306
|
-
|
307
|
-
await self._send_examples(examples_to_send, mapping.kind)
|
348
|
+
await self._send_examples(examples_to_send.get_examples(), mapping.kind)
|
308
349
|
|
309
350
|
return CalculationResult(
|
310
351
|
EntitySelectorDiff(passed=passed_entities, failed=failed_entities),
|
311
352
|
errors,
|
312
|
-
|
353
|
+
misconfigured_entity_keys=entity_misconfigurations,
|
313
354
|
)
|
@@ -32,9 +32,21 @@ from port_ocean.core.ocean_types import (
|
|
32
32
|
RAW_ITEM,
|
33
33
|
CalculationResult,
|
34
34
|
)
|
35
|
-
from port_ocean.core.utils.utils import
|
36
|
-
|
37
|
-
|
35
|
+
from port_ocean.core.utils.utils import (
|
36
|
+
resolve_entities_diff,
|
37
|
+
zip_and_sum,
|
38
|
+
gather_and_split_errors_from_results,
|
39
|
+
)
|
40
|
+
from port_ocean.exceptions.core import (
|
41
|
+
IntegrationSubProcessFailedException,
|
42
|
+
OceanAbortException,
|
43
|
+
)
|
44
|
+
from port_ocean.helpers.metric.metric import (
|
45
|
+
MetricResourceKind,
|
46
|
+
SyncState,
|
47
|
+
MetricType,
|
48
|
+
MetricPhase,
|
49
|
+
)
|
38
50
|
from port_ocean.helpers.metric.utils import TimeMetric, TimeMetricWithResourceKind
|
39
51
|
from port_ocean.utils.ipc import FileIPC
|
40
52
|
|
@@ -101,10 +113,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
101
113
|
results = []
|
102
114
|
for task in fns:
|
103
115
|
if inspect.isasyncgenfunction(task):
|
104
|
-
logger.info(
|
116
|
+
logger.info(
|
117
|
+
f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
|
118
|
+
)
|
105
119
|
results.append(resync_generator_wrapper(task, resource_config.kind))
|
106
120
|
else:
|
107
|
-
logger.info(
|
121
|
+
logger.info(
|
122
|
+
f"Found sync function for {resource_config.kind} name: {task.__qualname__}"
|
123
|
+
)
|
108
124
|
task = typing.cast(Callable[[str], Awaitable[RAW_RESULT]], task)
|
109
125
|
tasks.append(resync_function_wrapper(task, resource_config.kind))
|
110
126
|
|
@@ -155,14 +171,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
155
171
|
{
|
156
172
|
"property": "$identifier",
|
157
173
|
"operator": "in",
|
158
|
-
"value": [entity.identifier for entity in entities]
|
174
|
+
"value": [entity.identifier for entity in entities],
|
159
175
|
},
|
160
176
|
{
|
161
177
|
"property": "$blueprint",
|
162
178
|
"operator": "=",
|
163
|
-
"value": entities[0].blueprint
|
164
|
-
}
|
165
|
-
]
|
179
|
+
"value": entities[0].blueprint,
|
180
|
+
},
|
181
|
+
],
|
166
182
|
}
|
167
183
|
|
168
184
|
async def _map_entities_compared_with_port(
|
@@ -174,7 +190,10 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
174
190
|
if not entities:
|
175
191
|
return []
|
176
192
|
|
177
|
-
if
|
193
|
+
if (
|
194
|
+
entities[0].is_using_search_identifier
|
195
|
+
or entities[0].is_using_search_relation
|
196
|
+
):
|
178
197
|
return entities
|
179
198
|
|
180
199
|
MIN_ENTITIES_TO_MAP = 10
|
@@ -186,21 +205,21 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
186
205
|
|
187
206
|
# Process entities in batches
|
188
207
|
for start_index in range(0, len(entities), BATCH_SIZE):
|
189
|
-
entities_batch = entities[start_index:start_index + BATCH_SIZE]
|
208
|
+
entities_batch = entities[start_index : start_index + BATCH_SIZE]
|
190
209
|
batch_results = await self._fetch_entities_batch_from_port(
|
191
|
-
entities_batch,
|
192
|
-
resource,
|
193
|
-
user_agent_type
|
210
|
+
entities_batch, resource, user_agent_type
|
194
211
|
)
|
195
212
|
entities_at_port_with_properties.extend(batch_results)
|
196
213
|
|
197
|
-
logger.info(
|
214
|
+
logger.info(
|
215
|
+
"Got entities from port with properties and relations",
|
216
|
+
port_entities=len(entities_at_port_with_properties),
|
217
|
+
)
|
198
218
|
|
199
219
|
if len(entities_at_port_with_properties) > 0:
|
200
220
|
return resolve_entities_diff(entities, entities_at_port_with_properties)
|
201
221
|
return entities
|
202
222
|
|
203
|
-
|
204
223
|
async def _fetch_entities_batch_from_port(
|
205
224
|
self,
|
206
225
|
entities_batch: list[Entity],
|
@@ -210,16 +229,18 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
210
229
|
query = self._construct_search_query_for_entities(entities_batch)
|
211
230
|
return await ocean.port_client.search_entities(
|
212
231
|
user_agent_type,
|
213
|
-
parameters_to_include=["blueprint", "identifier"]
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
]
|
220
|
-
|
232
|
+
parameters_to_include=["blueprint", "identifier"]
|
233
|
+
+ (["title"] if resource.port.entity.mappings.title != None else [])
|
234
|
+
+ (["team"] if resource.port.entity.mappings.team != None else [])
|
235
|
+
+ [
|
236
|
+
f"properties.{prop}"
|
237
|
+
for prop in resource.port.entity.mappings.properties
|
238
|
+
]
|
239
|
+
+ [
|
240
|
+
f"relations.{relation}"
|
241
|
+
for relation in resource.port.entity.mappings.relations
|
221
242
|
],
|
222
|
-
query=query
|
243
|
+
query=query,
|
223
244
|
)
|
224
245
|
|
225
246
|
async def _register_resource_raw(
|
@@ -228,7 +249,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
228
249
|
results: list[dict[Any, Any]],
|
229
250
|
user_agent_type: UserAgentType,
|
230
251
|
parse_all: bool = False,
|
231
|
-
send_raw_data_examples_amount: int = 0
|
252
|
+
send_raw_data_examples_amount: int = 0,
|
232
253
|
) -> CalculationResult:
|
233
254
|
objects_diff = await self._calculate_raw(
|
234
255
|
[(resource, results)], parse_all, send_raw_data_examples_amount
|
@@ -236,8 +257,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
236
257
|
|
237
258
|
ocean.metrics.inc_metric(
|
238
259
|
name=MetricType.OBJECT_COUNT_NAME,
|
239
|
-
labels=[
|
240
|
-
|
260
|
+
labels=[
|
261
|
+
ocean.metrics.current_resource_kind(),
|
262
|
+
MetricPhase.TRANSFORM,
|
263
|
+
MetricPhase.TransformResult.FAILED,
|
264
|
+
],
|
265
|
+
value=len(objects_diff[0].entity_selector_diff.failed),
|
241
266
|
)
|
242
267
|
|
243
268
|
modified_objects = []
|
@@ -247,43 +272,67 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
247
272
|
changed_entities = await self._map_entities_compared_with_port(
|
248
273
|
objects_diff[0].entity_selector_diff.passed,
|
249
274
|
resource,
|
250
|
-
user_agent_type
|
275
|
+
user_agent_type,
|
251
276
|
)
|
252
277
|
if changed_entities:
|
253
|
-
logger.info(
|
254
|
-
|
278
|
+
logger.info(
|
279
|
+
"Upserting changed entities",
|
280
|
+
changed_entities=len(changed_entities),
|
281
|
+
total_entities=len(objects_diff[0].entity_selector_diff.passed),
|
282
|
+
)
|
255
283
|
ocean.metrics.inc_metric(
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
284
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
285
|
+
labels=[
|
286
|
+
ocean.metrics.current_resource_kind(),
|
287
|
+
MetricPhase.LOAD,
|
288
|
+
MetricPhase.LoadResult.SKIPPED,
|
289
|
+
],
|
290
|
+
value=len(objects_diff[0].entity_selector_diff.passed)
|
291
|
+
- len(changed_entities),
|
292
|
+
)
|
260
293
|
await self.entities_state_applier.upsert(
|
261
294
|
changed_entities, user_agent_type
|
262
295
|
)
|
263
296
|
|
264
297
|
else:
|
265
|
-
logger.info(
|
298
|
+
logger.info(
|
299
|
+
"Entities in batch didn't changed since last sync, skipping",
|
300
|
+
total_entities=len(objects_diff[0].entity_selector_diff.passed),
|
301
|
+
)
|
266
302
|
ocean.metrics.inc_metric(
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
303
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
304
|
+
labels=[
|
305
|
+
ocean.metrics.current_resource_kind(),
|
306
|
+
MetricPhase.LOAD,
|
307
|
+
MetricPhase.LoadResult.SKIPPED,
|
308
|
+
],
|
309
|
+
value=len(objects_diff[0].entity_selector_diff.passed),
|
310
|
+
)
|
311
|
+
modified_objects = [
|
312
|
+
ocean.port_client._reduce_entity(entity)
|
313
|
+
for entity in objects_diff[0].entity_selector_diff.passed
|
314
|
+
]
|
272
315
|
except Exception as e:
|
273
|
-
logger.warning(
|
316
|
+
logger.warning(
|
317
|
+
f"Failed to resolve batch entities with Port, falling back to upserting all entities: {str(e)}"
|
318
|
+
)
|
274
319
|
modified_objects = await self.entities_state_applier.upsert(
|
275
320
|
objects_diff[0].entity_selector_diff.passed, user_agent_type
|
276
|
-
|
321
|
+
)
|
277
322
|
else:
|
278
|
-
|
279
|
-
|
280
|
-
|
323
|
+
modified_objects = await self.entities_state_applier.upsert(
|
324
|
+
objects_diff[0].entity_selector_diff.passed, user_agent_type
|
325
|
+
)
|
281
326
|
|
282
327
|
return CalculationResult(
|
283
|
-
number_of_transformed_entities=len(
|
284
|
-
|
328
|
+
number_of_transformed_entities=len(
|
329
|
+
objects_diff[0].entity_selector_diff.passed
|
330
|
+
),
|
331
|
+
entity_selector_diff=objects_diff[0].entity_selector_diff._replace(
|
332
|
+
passed=modified_objects
|
333
|
+
),
|
285
334
|
errors=objects_diff[0].errors,
|
286
|
-
|
335
|
+
misconfigured_entity_keys=objects_diff[0].misconfigured_entity_keys
|
287
336
|
)
|
288
337
|
|
289
338
|
async def _unregister_resource_raw(
|
@@ -334,17 +383,17 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
334
383
|
resource_config,
|
335
384
|
raw_results,
|
336
385
|
user_agent_type,
|
337
|
-
send_raw_data_examples_amount=send_raw_data_examples_amount
|
386
|
+
send_raw_data_examples_amount=send_raw_data_examples_amount,
|
338
387
|
)
|
339
388
|
errors.extend(calculation_result.errors)
|
340
389
|
passed_entities = list(calculation_result.entity_selector_diff.passed)
|
341
|
-
number_of_transformed_entities +=
|
390
|
+
number_of_transformed_entities += (
|
391
|
+
calculation_result.number_of_transformed_entities
|
392
|
+
)
|
342
393
|
logger.info(
|
343
394
|
f"Finished registering change for {len(raw_results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
|
344
395
|
)
|
345
396
|
|
346
|
-
|
347
|
-
|
348
397
|
for generator in async_generators:
|
349
398
|
try:
|
350
399
|
async for items in generator:
|
@@ -358,11 +407,15 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
358
407
|
resource_config,
|
359
408
|
items,
|
360
409
|
user_agent_type,
|
361
|
-
send_raw_data_examples_amount=send_raw_data_examples_amount
|
410
|
+
send_raw_data_examples_amount=send_raw_data_examples_amount,
|
411
|
+
)
|
412
|
+
passed_entities.extend(
|
413
|
+
calculation_result.entity_selector_diff.passed
|
362
414
|
)
|
363
|
-
passed_entities.extend(calculation_result.entity_selector_diff.passed)
|
364
415
|
errors.extend(calculation_result.errors)
|
365
|
-
number_of_transformed_entities +=
|
416
|
+
number_of_transformed_entities += (
|
417
|
+
calculation_result.number_of_transformed_entities
|
418
|
+
)
|
366
419
|
except* OceanAbortException as error:
|
367
420
|
ocean.metrics.sync_state = SyncState.FAILED
|
368
421
|
errors.append(error)
|
@@ -371,29 +424,40 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
371
424
|
f"Finished registering kind: {resource_config.kind}-{resource.resource.index} ,{len(passed_entities)} entities out of {number_of_raw_results} raw results"
|
372
425
|
)
|
373
426
|
|
374
|
-
|
375
427
|
ocean.metrics.set_metric(
|
376
428
|
name=MetricType.SUCCESS_NAME,
|
377
429
|
labels=[ocean.metrics.current_resource_kind(), MetricPhase.RESYNC],
|
378
|
-
value=int(not errors)
|
430
|
+
value=int(not errors),
|
379
431
|
)
|
380
432
|
|
381
433
|
ocean.metrics.inc_metric(
|
382
434
|
name=MetricType.OBJECT_COUNT_NAME,
|
383
|
-
labels=[
|
384
|
-
|
435
|
+
labels=[
|
436
|
+
ocean.metrics.current_resource_kind(),
|
437
|
+
MetricPhase.EXTRACT,
|
438
|
+
MetricPhase.ExtractResult.EXTRACTED,
|
439
|
+
],
|
440
|
+
value=number_of_raw_results,
|
385
441
|
)
|
386
442
|
|
387
443
|
ocean.metrics.inc_metric(
|
388
444
|
name=MetricType.OBJECT_COUNT_NAME,
|
389
|
-
labels=[
|
390
|
-
|
445
|
+
labels=[
|
446
|
+
ocean.metrics.current_resource_kind(),
|
447
|
+
MetricPhase.TRANSFORM,
|
448
|
+
MetricPhase.TransformResult.TRANSFORMED,
|
449
|
+
],
|
450
|
+
value=number_of_transformed_entities,
|
391
451
|
)
|
392
452
|
|
393
453
|
ocean.metrics.inc_metric(
|
394
454
|
name=MetricType.OBJECT_COUNT_NAME,
|
395
|
-
labels=[
|
396
|
-
|
455
|
+
labels=[
|
456
|
+
ocean.metrics.current_resource_kind(),
|
457
|
+
MetricPhase.TRANSFORM,
|
458
|
+
MetricPhase.TransformResult.FILTERED_OUT,
|
459
|
+
],
|
460
|
+
value=number_of_raw_results - number_of_transformed_entities,
|
397
461
|
)
|
398
462
|
|
399
463
|
return passed_entities, errors
|
@@ -440,7 +504,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
440
504
|
errors = sum(errors, [])
|
441
505
|
misconfigured_entity_keys = list(misconfigured_entity_keys)
|
442
506
|
|
443
|
-
|
444
507
|
if errors:
|
445
508
|
message = f"Failed to register {len(errors)} entities. Skipping delete phase due to incomplete state"
|
446
509
|
logger.error(message, exc_info=errors)
|
@@ -580,34 +643,54 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
580
643
|
user_agent_type,
|
581
644
|
)
|
582
645
|
|
583
|
-
async def sort_and_upsert_failed_entities(
|
646
|
+
async def sort_and_upsert_failed_entities(
|
647
|
+
self, user_agent_type: UserAgentType
|
648
|
+
) -> None:
|
584
649
|
try:
|
585
650
|
if not event.entity_topological_sorter.should_execute():
|
586
651
|
return None
|
587
|
-
logger.info(
|
652
|
+
logger.info(
|
653
|
+
f"Executings topological sort of {event.entity_topological_sorter.get_entities_count()} entities failed to upsert.",
|
654
|
+
failed_toupsert_entities_count=event.entity_topological_sorter.get_entities_count(),
|
655
|
+
)
|
588
656
|
|
589
657
|
for entity in event.entity_topological_sorter.get_entities():
|
590
|
-
await self.entities_state_applier.context.port_client.upsert_entity(
|
658
|
+
await self.entities_state_applier.context.port_client.upsert_entity(
|
659
|
+
entity,
|
660
|
+
event.port_app_config.get_port_request_options(),
|
661
|
+
user_agent_type,
|
662
|
+
should_raise=False,
|
663
|
+
)
|
591
664
|
|
592
665
|
except OceanAbortException as ocean_abort:
|
593
|
-
logger.info(
|
594
|
-
|
666
|
+
logger.info(
|
667
|
+
f"Failed topological sort of failed to upsert entites - trying to upsert unordered {event.entity_topological_sorter.get_entities_count()} entities.",
|
668
|
+
failed_topological_sort_entities_count=event.entity_topological_sorter.get_entities_count(),
|
669
|
+
)
|
670
|
+
if isinstance(ocean_abort.__cause__, CycleError):
|
595
671
|
for entity in event.entity_topological_sorter.get_entities(False):
|
596
|
-
await self.entities_state_applier.context.port_client.upsert_entity(
|
672
|
+
await self.entities_state_applier.context.port_client.upsert_entity(
|
673
|
+
entity,
|
674
|
+
event.port_app_config.get_port_request_options(),
|
675
|
+
user_agent_type,
|
676
|
+
should_raise=False,
|
677
|
+
)
|
597
678
|
|
598
|
-
def process_resource_in_subprocess(
|
679
|
+
def process_resource_in_subprocess(
|
680
|
+
self,
|
599
681
|
file_ipc_map: dict[str, FileIPC],
|
600
682
|
resource: ResourceConfig,
|
601
683
|
index: int,
|
602
684
|
user_agent_type: UserAgentType,
|
603
685
|
) -> None:
|
604
|
-
logger.info(
|
686
|
+
logger.info(
|
687
|
+
f"process started successfully for {resource.kind} with index {index}"
|
688
|
+
)
|
605
689
|
|
606
690
|
clear_http_client_context()
|
691
|
+
|
607
692
|
async def process_resource_task() -> None:
|
608
|
-
result = await self._process_resource(
|
609
|
-
resource, index, user_agent_type
|
610
|
-
)
|
693
|
+
result = await self._process_resource(resource, index, user_agent_type)
|
611
694
|
file_ipc_map["process_resource"].save(result)
|
612
695
|
file_ipc_map["topological_entities"].save(
|
613
696
|
event.entity_topological_sorter.entities
|
@@ -616,32 +699,17 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
616
699
|
asyncio.run(process_resource_task())
|
617
700
|
logger.info(f"Process finished for {resource.kind} with index {index}")
|
618
701
|
|
619
|
-
async def
|
620
|
-
|
621
|
-
|
622
|
-
id = uuid.uuid4()
|
623
|
-
logger.info(f"Starting subprocess with id {id}")
|
624
|
-
file_ipc_map = {
|
625
|
-
"process_resource": FileIPC(id, "process_resource",([],[IntegrationSubProcessFailedException(f"Subprocess failed for {resource.kind} with index {index}")])),
|
626
|
-
"topological_entities": FileIPC(id, "topological_entities",[]),
|
627
|
-
}
|
628
|
-
process = ProcessWrapper(target=self.process_resource_in_subprocess, args=(file_ipc_map,resource,index,user_agent_type))
|
629
|
-
process.start()
|
630
|
-
await process.join_async()
|
631
|
-
|
632
|
-
event.entity_topological_sorter.entities.extend(file_ipc_map["topological_entities"].load())
|
633
|
-
return file_ipc_map["process_resource"].load()
|
634
|
-
|
635
|
-
else:
|
636
|
-
return await self._process_resource(resource,index,user_agent_type)
|
637
|
-
|
638
|
-
async def _process_resource(self,resource: ResourceConfig, index: int, user_agent_type: UserAgentType)-> tuple[list[Entity], list[Exception]]:
|
702
|
+
async def _process_resource(
|
703
|
+
self, resource: ResourceConfig, index: int, user_agent_type: UserAgentType
|
704
|
+
) -> tuple[list[Entity], list[Exception]]:
|
639
705
|
# create resource context per resource kind, so resync method could have access to the resource
|
640
706
|
# config as we might have multiple resources in the same event
|
641
|
-
async with resource_context(resource,index):
|
707
|
+
async with resource_context(resource, index):
|
642
708
|
resource_kind_id = f"{resource.kind}-{index}"
|
643
709
|
ocean.metrics.sync_state = SyncState.SYNCING
|
644
|
-
await ocean.metrics.report_kind_sync_metrics(
|
710
|
+
await ocean.metrics.report_kind_sync_metrics(
|
711
|
+
kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint
|
712
|
+
)
|
645
713
|
|
646
714
|
task = asyncio.create_task(
|
647
715
|
self._register_in_batches(resource, user_agent_type)
|
@@ -652,22 +720,87 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
652
720
|
if ocean.metrics.sync_state != SyncState.FAILED:
|
653
721
|
ocean.metrics.sync_state = SyncState.COMPLETED
|
654
722
|
|
655
|
-
await ocean.metrics.send_metrics_to_webhook(
|
656
|
-
|
723
|
+
await ocean.metrics.send_metrics_to_webhook(kind=resource_kind_id)
|
724
|
+
await ocean.metrics.report_kind_sync_metrics(
|
725
|
+
kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint
|
657
726
|
)
|
658
|
-
await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint)
|
659
727
|
|
660
728
|
return kind_results
|
661
729
|
|
662
|
-
|
663
|
-
async def resync_reconciliation(
|
730
|
+
def resync_reconciliation_in_subprocess(
|
664
731
|
self,
|
732
|
+
file_ipc_map: dict[str, FileIPC],
|
665
733
|
creation_results: list[tuple[list[Entity], list[Exception]]],
|
666
734
|
did_fetched_current_state: bool,
|
667
735
|
user_agent_type: UserAgentType,
|
668
736
|
app_config: Any,
|
669
737
|
silent: bool = True,
|
670
738
|
) -> None:
|
739
|
+
logger.info("Resync reconciliation subprocess started successfully")
|
740
|
+
|
741
|
+
clear_http_client_context()
|
742
|
+
|
743
|
+
async def resync_reconciliation_task() -> None:
|
744
|
+
result = await self._resync_reconciliation(
|
745
|
+
creation_results,
|
746
|
+
did_fetched_current_state,
|
747
|
+
user_agent_type,
|
748
|
+
app_config,
|
749
|
+
silent,
|
750
|
+
)
|
751
|
+
file_ipc_map["resync_reconciliation"].save(result)
|
752
|
+
|
753
|
+
asyncio.run(resync_reconciliation_task())
|
754
|
+
logger.info("Resync reconciliation subprocess finished")
|
755
|
+
|
756
|
+
async def process_resource(
|
757
|
+
self, resource: ResourceConfig, index: int, user_agent_type: UserAgentType
|
758
|
+
) -> tuple[list[Entity], list[Exception]]:
|
759
|
+
with logger.contextualize(resource_kind=resource.kind, index=index):
|
760
|
+
if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:
|
761
|
+
id = uuid.uuid4()
|
762
|
+
logger.info(f"Starting subprocess with id {id}")
|
763
|
+
file_ipc_map = {
|
764
|
+
"process_resource": FileIPC(
|
765
|
+
str(id),
|
766
|
+
"process_resource",
|
767
|
+
(
|
768
|
+
[],
|
769
|
+
[
|
770
|
+
IntegrationSubProcessFailedException(
|
771
|
+
f"Subprocess failed for {resource.kind} with index {index}"
|
772
|
+
)
|
773
|
+
],
|
774
|
+
),
|
775
|
+
),
|
776
|
+
"topological_entities": FileIPC(
|
777
|
+
str(id), "topological_entities", []
|
778
|
+
),
|
779
|
+
}
|
780
|
+
process = ProcessWrapper(
|
781
|
+
target=self.process_resource_in_subprocess,
|
782
|
+
args=(file_ipc_map, resource, index, user_agent_type),
|
783
|
+
)
|
784
|
+
process.start()
|
785
|
+
await process.join_async()
|
786
|
+
|
787
|
+
event.entity_topological_sorter.entities.extend(
|
788
|
+
file_ipc_map["topological_entities"].load()
|
789
|
+
)
|
790
|
+
return file_ipc_map["process_resource"].load()
|
791
|
+
|
792
|
+
else:
|
793
|
+
return await self._process_resource(resource, index, user_agent_type)
|
794
|
+
|
795
|
+
@TimeMetricWithResourceKind(MetricPhase.RESYNC)
|
796
|
+
async def _resync_reconciliation(
|
797
|
+
self,
|
798
|
+
creation_results: list[tuple[list[Entity], list[Exception]]],
|
799
|
+
did_fetched_current_state: bool,
|
800
|
+
user_agent_type: UserAgentType,
|
801
|
+
app_config: Any,
|
802
|
+
silent: bool = True,
|
803
|
+
) -> bool:
|
671
804
|
"""Handle the reconciliation phase of the resync process.
|
672
805
|
|
673
806
|
This method handles:
|
@@ -716,13 +849,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
716
849
|
logger.info(
|
717
850
|
f"Running resync diff calculation, number of entities created during sync: {len(generated_entities)}"
|
718
851
|
)
|
719
|
-
entities_at_port = await ocean.port_client.search_entities(
|
720
|
-
user_agent_type
|
721
|
-
)
|
852
|
+
entities_at_port = await ocean.port_client.search_entities(user_agent_type)
|
722
853
|
|
723
854
|
await self.entities_state_applier.delete_diff(
|
724
855
|
{"before": entities_at_port, "after": generated_entities},
|
725
|
-
user_agent_type,
|
856
|
+
user_agent_type,
|
857
|
+
app_config.get_entity_deletion_threshold(),
|
726
858
|
)
|
727
859
|
|
728
860
|
logger.info("Resync finished successfully")
|
@@ -738,6 +870,47 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
738
870
|
|
739
871
|
return True
|
740
872
|
|
873
|
+
async def resync_reconciliation(
|
874
|
+
self,
|
875
|
+
creation_results: list[tuple[list[Entity], list[Exception]]],
|
876
|
+
did_fetched_current_state: bool,
|
877
|
+
user_agent_type: UserAgentType,
|
878
|
+
app_config: Any,
|
879
|
+
silent: bool = True,
|
880
|
+
) -> bool:
|
881
|
+
if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:
|
882
|
+
id = uuid.uuid4()
|
883
|
+
logger.info(f"Starting resync reconciliation in subprocess with id {id}")
|
884
|
+
|
885
|
+
file_ipc_map = {
|
886
|
+
"resync_reconciliation": FileIPC(
|
887
|
+
str(id), "resync_reconciliation", False
|
888
|
+
),
|
889
|
+
}
|
890
|
+
|
891
|
+
process = ProcessWrapper(
|
892
|
+
target=self.resync_reconciliation_in_subprocess,
|
893
|
+
args=(
|
894
|
+
file_ipc_map,
|
895
|
+
creation_results,
|
896
|
+
did_fetched_current_state,
|
897
|
+
user_agent_type,
|
898
|
+
app_config,
|
899
|
+
silent,
|
900
|
+
),
|
901
|
+
)
|
902
|
+
process.start()
|
903
|
+
await process.join_async()
|
904
|
+
|
905
|
+
return file_ipc_map["resync_reconciliation"].load()
|
906
|
+
else:
|
907
|
+
return await self._resync_reconciliation(
|
908
|
+
creation_results,
|
909
|
+
did_fetched_current_state,
|
910
|
+
user_agent_type,
|
911
|
+
app_config,
|
912
|
+
silent,
|
913
|
+
)
|
741
914
|
|
742
915
|
@TimeMetric(MetricPhase.RESYNC)
|
743
916
|
async def sync_raw_all(
|
@@ -773,8 +946,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
773
946
|
)
|
774
947
|
logger.info(f"Resync will use the following mappings: {app_config.dict()}")
|
775
948
|
|
776
|
-
kinds = [
|
777
|
-
|
949
|
+
kinds = [
|
950
|
+
f"{resource.kind}-{index}"
|
951
|
+
for index, resource in enumerate(app_config.resources)
|
952
|
+
]
|
953
|
+
blueprints = [
|
954
|
+
resource.port.entity.mappings.blueprint
|
955
|
+
for resource in app_config.resources
|
956
|
+
]
|
778
957
|
ocean.metrics.initialize_metrics(kinds)
|
779
958
|
await ocean.metrics.report_sync_metrics(kinds=kinds, blueprints=blueprints)
|
780
959
|
|
@@ -799,13 +978,19 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
799
978
|
|
800
979
|
creation_results: list[tuple[list[Entity], list[Exception]]] = []
|
801
980
|
|
802
|
-
multiprocessing.set_start_method(
|
981
|
+
multiprocessing.set_start_method("fork", True)
|
803
982
|
try:
|
804
|
-
for index,resource in enumerate(app_config.resources):
|
805
|
-
logger.info(
|
806
|
-
|
983
|
+
for index, resource in enumerate(app_config.resources):
|
984
|
+
logger.info(
|
985
|
+
f"Starting processing resource {resource.kind} with index {index}"
|
986
|
+
)
|
987
|
+
creation_results.append(
|
988
|
+
await self.process_resource(resource, index, user_agent_type)
|
989
|
+
)
|
807
990
|
except asyncio.CancelledError as e:
|
808
|
-
logger.warning(
|
991
|
+
logger.warning(
|
992
|
+
"Resync aborted successfully, skipping delete phase. This leads to an incomplete state"
|
993
|
+
)
|
809
994
|
raise
|
810
995
|
else:
|
811
996
|
success = await self.resync_reconciliation(
|
@@ -813,11 +998,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
813
998
|
did_fetched_current_state,
|
814
999
|
user_agent_type,
|
815
1000
|
app_config,
|
816
|
-
silent
|
1001
|
+
silent,
|
1002
|
+
)
|
1003
|
+
await ocean.metrics.report_sync_metrics(
|
1004
|
+
kinds=[MetricResourceKind.RECONCILIATION]
|
817
1005
|
)
|
818
|
-
await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RECONCILIATION])
|
819
1006
|
return success
|
820
1007
|
finally:
|
821
1008
|
await ocean.app.cache_provider.clear()
|
822
|
-
if
|
1009
|
+
if (
|
1010
|
+
ocean.app.process_execution_mode
|
1011
|
+
== ProcessExecutionMode.multi_process
|
1012
|
+
):
|
823
1013
|
ocean.metrics.cleanup_prometheus_metrics()
|
port_ocean/core/ocean_types.py
CHANGED
@@ -42,7 +42,7 @@ class CalculationResult(NamedTuple):
|
|
42
42
|
entity_selector_diff: EntitySelectorDiff
|
43
43
|
errors: list[Exception]
|
44
44
|
number_of_transformed_entities: int = 0
|
45
|
-
|
45
|
+
misconfigured_entity_keys: dict[str, str] = field(default_factory=dict)
|
46
46
|
|
47
47
|
|
48
48
|
class IntegrationEventsCallbacks(TypedDict):
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import Any
|
1
|
+
from typing import cast, Any
|
2
2
|
from unittest.mock import AsyncMock, Mock
|
3
3
|
from loguru import logger
|
4
4
|
import pytest
|
@@ -10,6 +10,7 @@ from port_ocean.core.handlers.entity_processor.jq_entity_processor import (
|
|
10
10
|
)
|
11
11
|
from port_ocean.core.ocean_types import CalculationResult
|
12
12
|
from port_ocean.exceptions.core import EntityProcessorException
|
13
|
+
from unittest.mock import patch
|
13
14
|
|
14
15
|
|
15
16
|
@pytest.mark.asyncio
|
@@ -296,15 +297,23 @@ class TestJQEntityProcessor:
|
|
296
297
|
},
|
297
298
|
{"foo": "bar", "baz": "bazbar", "bar": {"foobar": "foobar"}},
|
298
299
|
]
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
300
|
+
with patch(
|
301
|
+
"port_ocean.core.handlers.entity_processor.jq_entity_processor.JQEntityProcessor._send_examples"
|
302
|
+
) as mock_send_examples:
|
303
|
+
result = await mocked_processor._parse_items(
|
304
|
+
mapping, raw_results, send_raw_data_examples_amount=1
|
305
|
+
)
|
306
|
+
assert len(result.misconfigured_entity_keys) > 0
|
307
|
+
assert len(result.misconfigured_entity_keys) == 4
|
308
|
+
assert result.misconfigured_entity_keys == {
|
309
|
+
"identifier": ".ark",
|
310
|
+
"description": ".bazbar",
|
311
|
+
"url": ".foobar",
|
312
|
+
"defaultBranch": ".bar.baz",
|
313
|
+
}
|
314
|
+
assert mock_send_examples.await_args is not None, "mock was not awaited"
|
315
|
+
args, _ = mock_send_examples.await_args
|
316
|
+
assert len(cast(list[Any], args[0])) > 0
|
308
317
|
|
309
318
|
async def test_parse_items_empty_required(
|
310
319
|
self, mocked_processor: JQEntityProcessor
|
@@ -324,15 +333,15 @@ class TestJQEntityProcessor:
|
|
324
333
|
{"foo": "identifierMapped", "bar": ""},
|
325
334
|
]
|
326
335
|
result = await mocked_processor._parse_items(mapping, raw_results)
|
327
|
-
assert "identifier" not in result.
|
328
|
-
assert "blueprint" not in result.
|
336
|
+
assert "identifier" not in result.misconfigured_entity_keys
|
337
|
+
assert "blueprint" not in result.misconfigured_entity_keys
|
329
338
|
|
330
339
|
raw_results = [
|
331
340
|
{"foo": "identifierMapped", "bar": None},
|
332
341
|
{"foo": None, "bar": ""},
|
333
342
|
]
|
334
343
|
result = await mocked_processor._parse_items(mapping, raw_results)
|
335
|
-
assert result.
|
344
|
+
assert result.misconfigured_entity_keys == {
|
336
345
|
"identifier": ".foo",
|
337
346
|
"blueprint": ".bar",
|
338
347
|
}
|
@@ -334,7 +334,7 @@ async def test_parse_raw_event_results_to_entities_creation(
|
|
334
334
|
calculation_result = CalculationResult(
|
335
335
|
entity_selector_diff=EntitySelectorDiff(passed=[entity], failed=[]),
|
336
336
|
errors=[],
|
337
|
-
|
337
|
+
misconfigured_entity_keys={},
|
338
338
|
)
|
339
339
|
mock_live_events_mixin.entity_processor.parse_items.return_value = (
|
340
340
|
calculation_result
|
@@ -361,7 +361,7 @@ async def test_parse_raw_event_results_to_entities_deletion(
|
|
361
361
|
calculation_result = CalculationResult(
|
362
362
|
entity_selector_diff=EntitySelectorDiff(passed=[entity], failed=[]),
|
363
363
|
errors=[],
|
364
|
-
|
364
|
+
misconfigured_entity_keys={},
|
365
365
|
)
|
366
366
|
mock_live_events_mixin.entity_processor.parse_items.return_value = (
|
367
367
|
calculation_result
|
@@ -113,7 +113,7 @@ async def test_sync_raw_mixin_self_dependency(
|
|
113
113
|
calc_result_mock.number_of_transformed_entities = len(
|
114
114
|
entities
|
115
115
|
) # Add this to match real behavior
|
116
|
-
calc_result_mock.
|
116
|
+
calc_result_mock.misconfigured_entity_keys = {} # Add this to match real behavior
|
117
117
|
|
118
118
|
mock_sync_raw_mixin.entity_processor.parse_items = AsyncMock(return_value=calc_result_mock) # type: ignore
|
119
119
|
|
@@ -234,7 +234,7 @@ async def test_sync_raw_mixin_circular_dependency(
|
|
234
234
|
calc_result_mock.number_of_transformed_entities = len(
|
235
235
|
entities
|
236
236
|
) # Add this to match real behavior
|
237
|
-
calc_result_mock.
|
237
|
+
calc_result_mock.misconfigured_entity_keys = {} # Add this to match real behavior
|
238
238
|
|
239
239
|
mock_sync_raw_mixin.entity_processor.parse_items = AsyncMock(return_value=calc_result_mock) # type: ignore
|
240
240
|
|
@@ -378,7 +378,7 @@ async def test_sync_raw_mixin_dependency(
|
|
378
378
|
calc_result_mock.number_of_transformed_entities = len(
|
379
379
|
entities
|
380
380
|
) # Add this to match real behavior
|
381
|
-
calc_result_mock.
|
381
|
+
calc_result_mock.misconfigured_entity_keys = {} # Add this to match real behavior
|
382
382
|
|
383
383
|
# Mock the parse_items method to return our realistic mock
|
384
384
|
mock_sync_raw_mixin.entity_processor.parse_items = AsyncMock(return_value=calc_result_mock) # type: ignore
|
@@ -773,7 +773,7 @@ class CalculationResult:
|
|
773
773
|
entity_selector_diff: EntitySelectorDiff
|
774
774
|
errors: List[Any]
|
775
775
|
misconfigurations: List[Any]
|
776
|
-
|
776
|
+
misconfigured_entity_keys: Optional[List[Any]] = None
|
777
777
|
|
778
778
|
|
779
779
|
@pytest.mark.asyncio
|
@@ -782,7 +782,7 @@ async def test_register_resource_raw_no_changes_upsert_not_called_entitiy_is_ret
|
|
782
782
|
mock_port_app_config: PortAppConfig,
|
783
783
|
) -> None:
|
784
784
|
entity = Entity(identifier="1", blueprint="service")
|
785
|
-
mock_sync_raw_mixin._calculate_raw = AsyncMock(return_value=[CalculationResult(entity_selector_diff=EntitySelectorDiff(passed=[entity], failed=[]), errors=[], misconfigurations=[],
|
785
|
+
mock_sync_raw_mixin._calculate_raw = AsyncMock(return_value=[CalculationResult(entity_selector_diff=EntitySelectorDiff(passed=[entity], failed=[]), errors=[], misconfigurations=[], misconfigured_entity_keys=[])]) # type: ignore
|
786
786
|
mock_sync_raw_mixin._map_entities_compared_with_port = AsyncMock(return_value=([])) # type: ignore
|
787
787
|
mock_sync_raw_mixin.entities_state_applier.upsert = AsyncMock() # type: ignore
|
788
788
|
|
@@ -809,7 +809,7 @@ async def test_register_resource_raw_with_changes_upsert_called_and_entities_are
|
|
809
809
|
mock_port_app_config: PortAppConfig,
|
810
810
|
) -> None:
|
811
811
|
entity = Entity(identifier="1", blueprint="service")
|
812
|
-
mock_sync_raw_mixin._calculate_raw = AsyncMock(return_value=[CalculationResult(entity_selector_diff=EntitySelectorDiff(passed=[entity], failed=[]), errors=[], misconfigurations=[],
|
812
|
+
mock_sync_raw_mixin._calculate_raw = AsyncMock(return_value=[CalculationResult(entity_selector_diff=EntitySelectorDiff(passed=[entity], failed=[]), errors=[], misconfigurations=[], misconfigured_entity_keys=[])]) # type: ignore
|
813
813
|
mock_sync_raw_mixin._map_entities_compared_with_port = AsyncMock(return_value=([entity])) # type: ignore
|
814
814
|
mock_sync_raw_mixin.entities_state_applier.upsert = AsyncMock(return_value=[entity]) # type: ignore
|
815
815
|
|
@@ -836,7 +836,7 @@ async def test_register_resource_raw_with_errors(
|
|
836
836
|
) -> None:
|
837
837
|
failed_entity = Entity(identifier="1", blueprint="service")
|
838
838
|
error = Exception("Test error")
|
839
|
-
mock_sync_raw_mixin._calculate_raw = AsyncMock(return_value=[CalculationResult(entity_selector_diff=EntitySelectorDiff(passed=[], failed=[failed_entity]), errors=[error], misconfigurations=[],
|
839
|
+
mock_sync_raw_mixin._calculate_raw = AsyncMock(return_value=[CalculationResult(entity_selector_diff=EntitySelectorDiff(passed=[], failed=[failed_entity]), errors=[error], misconfigurations=[], misconfigured_entity_keys=[])]) # type: ignore
|
840
840
|
mock_sync_raw_mixin._map_entities_compared_with_port = AsyncMock(return_value=([])) # type: ignore
|
841
841
|
mock_sync_raw_mixin.entities_state_applier.upsert = AsyncMock() # type: ignore
|
842
842
|
|
@@ -873,7 +873,7 @@ async def test_register_resource_raw_skip_event_type_http_request_upsert_called_
|
|
873
873
|
entity_selector_diff=EntitySelectorDiff(passed=[entity], failed=[]),
|
874
874
|
errors=[],
|
875
875
|
misconfigurations=[],
|
876
|
-
|
876
|
+
misconfigured_entity_keys=[],
|
877
877
|
)
|
878
878
|
mock_sync_raw_mixin._calculate_raw = AsyncMock(return_value=[calculation_result]) # type: ignore
|
879
879
|
mock_sync_raw_mixin._map_entities_compared_with_port = AsyncMock() # type: ignore
|
@@ -1,11 +1,12 @@
|
|
1
|
-
integrations/_infra/Dockerfile.Deb,sha256=
|
1
|
+
integrations/_infra/Dockerfile.Deb,sha256=YKr_k6A_o7zEd15WYxYMdv9q9vnVZ4Yb2yyj8rb6ihQ,1998
|
2
2
|
integrations/_infra/Dockerfile.alpine,sha256=7E4Sb-8supsCcseerHwTkuzjHZoYcaHIyxiBZ-wewo0,3482
|
3
3
|
integrations/_infra/Dockerfile.base.builder,sha256=ESe1PKC6itp_AuXawbLI75k1Kruny6NTANaTinxOgVs,743
|
4
4
|
integrations/_infra/Dockerfile.base.runner,sha256=uAcs2IsxrAAUHGXt_qULA5INr-HFguf5a5fCKiqEzbY,384
|
5
5
|
integrations/_infra/Dockerfile.dockerignore,sha256=CM1Fxt3I2AvSvObuUZRmy5BNLSGC7ylnbpWzFgD4cso,1163
|
6
|
-
integrations/_infra/Dockerfile.local,sha256=
|
6
|
+
integrations/_infra/Dockerfile.local,sha256=v-K0jgDj5vWyF4HCL8-wluUPB04pf9ycEjusRBmIVnc,1527
|
7
7
|
integrations/_infra/Makefile,sha256=YgLKvuF_Dw4IA7X98Nus6zIW_3cJ60M1QFGs3imj5c4,2430
|
8
|
-
integrations/_infra/
|
8
|
+
integrations/_infra/README.md,sha256=ZtJFSMCTU5zTeM8ddRuW1ZL1ga8z7Ic2F3mxmgOSjgo,1195
|
9
|
+
integrations/_infra/entry_local.sh,sha256=Sn2TexTEpruH2ixIAGsk-fZV6Y7pT3jd2Pi9TxBeFuw,633
|
9
10
|
integrations/_infra/grpcio.sh,sha256=m924poYznoRZ6Tt7Ct8Cs5AV_cmmOx598yIZ3z4DvZE,616
|
10
11
|
integrations/_infra/init.sh,sha256=nN8lTrOhB286UfFvD6sJ9YJ-9asT9zVSddQB-RAb7Z4,99
|
11
12
|
port_ocean/__init__.py,sha256=uMpjg5d_cXgnyCxA_LmICR8zqBmC6Fe9Ivu9hcvJ7EY,313
|
@@ -96,11 +97,11 @@ port_ocean/core/handlers/entities_state_applier/__init__.py,sha256=kgLZDCeCEzi4r
|
|
96
97
|
port_ocean/core/handlers/entities_state_applier/base.py,sha256=5wHL0icfFAYRPqk8iV_wN49GdJ3aRUtO8tumSxBi4Wo,2268
|
97
98
|
port_ocean/core/handlers/entities_state_applier/port/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
98
99
|
port_ocean/core/handlers/entities_state_applier/port/applier.py,sha256=NsLW35H1dhP5FFD7kHZxA2ndSBcD_-btan7qNWfvWrs,6683
|
99
|
-
port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha256=
|
100
|
+
port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha256=XKDIexoDPVWT2-Y1UTKqJJyRUWC3qiHKQD8bozIrni4,2038
|
100
101
|
port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=lyv6xKzhYfd6TioUgR3AVRSJqj7JpAaj1LxxU2xAqeo,1720
|
101
102
|
port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
|
102
103
|
port_ocean/core/handlers/entity_processor/base.py,sha256=PsnpNRqjHth9xwOvDRe7gKu8cjnVV0XGmTIHGvOelX0,1867
|
103
|
-
port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=
|
104
|
+
port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=GxK5BM0yzAoeYTK2-ALKh5qlAe9pebcN7vZfQCZjksM,12787
|
104
105
|
port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
|
105
106
|
port_ocean/core/handlers/port_app_config/api.py,sha256=r_Th66NEw38IpRdnXZcRvI8ACfvxW_A6V62WLwjWXlQ,1044
|
106
107
|
port_ocean/core/handlers/port_app_config/base.py,sha256=Sup4-X_a7JGa27rMy_OgqGIjFHMlKBpKevicaK3AeHU,2919
|
@@ -121,10 +122,10 @@ port_ocean/core/integrations/mixins/events.py,sha256=2L7P3Jhp8XBqddh2_o9Cn4N261n
|
|
121
122
|
port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
|
122
123
|
port_ocean/core/integrations/mixins/live_events.py,sha256=zM24dhNc7uHx9XYZ6toVhDADPA90EnpOmZxgDegFZbA,4196
|
123
124
|
port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
|
124
|
-
port_ocean/core/integrations/mixins/sync_raw.py,sha256=
|
125
|
+
port_ocean/core/integrations/mixins/sync_raw.py,sha256=AzlOARxsKY1YYQ6xhjYiDD-vELkci0Vz7THw8hDVgAU,38522
|
125
126
|
port_ocean/core/integrations/mixins/utils.py,sha256=N76dNu1Y6UEg0_pcSdF4RO6dQIZ8EBfX3xMelgWdMxM,3779
|
126
127
|
port_ocean/core/models.py,sha256=DNbKpStMINI2lIekKprTqBevqkw_wFuFayN19w1aDfQ,2893
|
127
|
-
port_ocean/core/ocean_types.py,sha256=
|
128
|
+
port_ocean/core/ocean_types.py,sha256=bkLlTd8XfJK6_JDl0eXUHfE_NygqgiInSMwJ4YJH01Q,1399
|
128
129
|
port_ocean/core/utils/entity_topological_sorter.py,sha256=MDUjM6OuDy4Xj68o-7InNN0w1jqjxeDfeY8U02vySNI,3081
|
129
130
|
port_ocean/core/utils/utils.py,sha256=XJ6ZZBR5hols19TcX4Bh49ygSNhPt3MLncLR-g41GTA,6858
|
130
131
|
port_ocean/debug_cli.py,sha256=gHrv-Ey3cImKOcGZpjoHlo4pa_zfmyOl6TUM4o9VtcA,96
|
@@ -166,9 +167,9 @@ port_ocean/tests/conftest.py,sha256=JXASSS0IY0nnR6bxBflhzxS25kf4iNaABmThyZ0mZt8,
|
|
166
167
|
port_ocean/tests/core/conftest.py,sha256=7K_M1--wQ08VmiQRB0vo1nst2X00cwsuBS5UfERsnG8,7589
|
167
168
|
port_ocean/tests/core/defaults/test_common.py,sha256=sR7RqB3ZYV6Xn6NIg-c8k5K6JcGsYZ2SCe_PYX5vLYM,5560
|
168
169
|
port_ocean/tests/core/handlers/entities_state_applier/test_applier.py,sha256=_CZyViY9_gnxjY6ogWcDdmEDuejvpALogf9ESjVAwFY,10675
|
169
|
-
port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=
|
170
|
-
port_ocean/tests/core/handlers/mixins/test_live_events.py,sha256=
|
171
|
-
port_ocean/tests/core/handlers/mixins/test_sync_raw.py,sha256
|
170
|
+
port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=AyzLclJFKz8YzSi2hiFYVdXRzCFmqQwTxnwNFTo9roU,14091
|
171
|
+
port_ocean/tests/core/handlers/mixins/test_live_events.py,sha256=6yUsYooBYchiZP_eYa8PN1IhiztJShBdPguoseyNMzY,12482
|
172
|
+
port_ocean/tests/core/handlers/mixins/test_sync_raw.py,sha256=-Jd2rUG63fZM8LuyKtCp1tt4WEqO2m5woESjs1c91sU,44428
|
172
173
|
port_ocean/tests/core/handlers/port_app_config/test_api.py,sha256=eJZ6SuFBLz71y4ca3DNqKag6d6HUjNJS0aqQPwiLMTI,1999
|
173
174
|
port_ocean/tests/core/handlers/port_app_config/test_base.py,sha256=hSh556bJM9zuELwhwnyKSfd9z06WqWXIfe-6hCl5iKI,9799
|
174
175
|
port_ocean/tests/core/handlers/queue/test_local_queue.py,sha256=9Ly0HzZXbs6Rbl_bstsIdInC3h2bgABU3roP9S_PnJM,2582
|
@@ -202,8 +203,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
|
|
202
203
|
port_ocean/utils/signal.py,sha256=mMVq-1Ab5YpNiqN4PkiyTGlV_G0wkUDMMjTZp5z3pb0,1514
|
203
204
|
port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
|
204
205
|
port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
|
205
|
-
port_ocean-0.
|
206
|
-
port_ocean-0.
|
207
|
-
port_ocean-0.
|
208
|
-
port_ocean-0.
|
209
|
-
port_ocean-0.
|
206
|
+
port_ocean-0.25.2.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
207
|
+
port_ocean-0.25.2.dist-info/METADATA,sha256=ruhq57S2upv-BOr7YML2ybb1fCrmXu6PHV_VTLTQnu4,6855
|
208
|
+
port_ocean-0.25.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
209
|
+
port_ocean-0.25.2.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
|
210
|
+
port_ocean-0.25.2.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|