port-ocean 0.30.3__py3-none-any.whl → 0.30.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- port_ocean/clients/port/mixins/entities.py +35 -26
- {port_ocean-0.30.3.dist-info → port_ocean-0.30.4.dist-info}/METADATA +1 -1
- {port_ocean-0.30.3.dist-info → port_ocean-0.30.4.dist-info}/RECORD +6 -6
- {port_ocean-0.30.3.dist-info → port_ocean-0.30.4.dist-info}/LICENSE.md +0 -0
- {port_ocean-0.30.3.dist-info → port_ocean-0.30.4.dist-info}/WHEEL +0 -0
- {port_ocean-0.30.3.dist-info → port_ocean-0.30.4.dist-info}/entry_points.txt +0 -0
|
@@ -262,34 +262,36 @@ class EntityClientMixin:
|
|
|
262
262
|
}
|
|
263
263
|
error_entities = {error["index"]: error for error in result.get("errors", [])}
|
|
264
264
|
|
|
265
|
+
ocean.metrics.inc_metric(
|
|
266
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
|
267
|
+
labels=[
|
|
268
|
+
ocean.metrics.current_resource_kind(),
|
|
269
|
+
MetricPhase.LOAD,
|
|
270
|
+
MetricPhase.LoadResult.LOADED,
|
|
271
|
+
],
|
|
272
|
+
value=len(successful_entities),
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
ocean.metrics.inc_metric(
|
|
276
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
|
277
|
+
labels=[
|
|
278
|
+
ocean.metrics.current_resource_kind(),
|
|
279
|
+
MetricPhase.LOAD,
|
|
280
|
+
MetricPhase.LoadResult.FAILED,
|
|
281
|
+
],
|
|
282
|
+
value=len(error_entities),
|
|
283
|
+
)
|
|
284
|
+
|
|
265
285
|
batch_results: list[tuple[bool | None, Entity]] = []
|
|
266
286
|
for entity_index, original_entity in index_to_entity.items():
|
|
267
287
|
reduced_entity = self._reduce_entity(original_entity)
|
|
268
288
|
if entity_index in successful_entities:
|
|
269
|
-
ocean.metrics.inc_metric(
|
|
270
|
-
name=MetricType.OBJECT_COUNT_NAME,
|
|
271
|
-
labels=[
|
|
272
|
-
ocean.metrics.current_resource_kind(),
|
|
273
|
-
MetricPhase.LOAD,
|
|
274
|
-
MetricPhase.LoadResult.LOADED,
|
|
275
|
-
],
|
|
276
|
-
value=1,
|
|
277
|
-
)
|
|
278
289
|
success_entity = successful_entities[entity_index]
|
|
279
290
|
# Create a copy of the original entity with the new identifier
|
|
280
291
|
updated_entity = reduced_entity.copy()
|
|
281
292
|
updated_entity.identifier = success_entity["identifier"]
|
|
282
293
|
batch_results.append((True, updated_entity))
|
|
283
294
|
elif entity_index in error_entities:
|
|
284
|
-
ocean.metrics.inc_metric(
|
|
285
|
-
name=MetricType.OBJECT_COUNT_NAME,
|
|
286
|
-
labels=[
|
|
287
|
-
ocean.metrics.current_resource_kind(),
|
|
288
|
-
MetricPhase.LOAD,
|
|
289
|
-
MetricPhase.LoadResult.FAILED,
|
|
290
|
-
],
|
|
291
|
-
value=1,
|
|
292
|
-
)
|
|
293
295
|
error = error_entities[entity_index]
|
|
294
296
|
if (
|
|
295
297
|
error.get("identifier") == "unknown"
|
|
@@ -357,16 +359,23 @@ class EntityClientMixin:
|
|
|
357
359
|
blueprint = entities[0].blueprint
|
|
358
360
|
|
|
359
361
|
identifier_counts = Counter((e.blueprint, e.identifier) for e in entities)
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
362
|
+
duplicates: dict[tuple[str, str], int] = {
|
|
363
|
+
key: cnt for key, cnt in identifier_counts.items() if cnt > 1
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
if duplicates:
|
|
367
|
+
# Total number of *extra* occurrences beyond the first appearance of each entity
|
|
368
|
+
duplicate_count = sum(cnt - 1 for cnt in duplicates.values())
|
|
369
|
+
|
|
370
|
+
# Show up to 5 example (blueprint, identifier) pairs to avoid noisy logs
|
|
371
|
+
duplicate_examples = list(duplicates)[:5]
|
|
363
372
|
|
|
364
|
-
if duplicate_count:
|
|
365
|
-
duplicate_examples = [
|
|
366
|
-
key for key, cnt in identifier_counts.items() if cnt > 1
|
|
367
|
-
][:5]
|
|
368
373
|
logger.warning(
|
|
369
|
-
|
|
374
|
+
"Detected duplicate entities (by blueprint and identifier) that may not be ingested because an identical identifier existed",
|
|
375
|
+
extra={
|
|
376
|
+
"duplicate_examples": duplicate_examples,
|
|
377
|
+
"duplicate_count": duplicate_count,
|
|
378
|
+
},
|
|
370
379
|
)
|
|
371
380
|
|
|
372
381
|
bulk_size = self.calculate_entities_batch_size(entities)
|
|
@@ -61,7 +61,7 @@ port_ocean/clients/port/client.py,sha256=LHR6zKgCCCyhe3aPWH0kRFYS02BN-lIDZMPQbaz
|
|
|
61
61
|
port_ocean/clients/port/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
62
62
|
port_ocean/clients/port/mixins/actions.py,sha256=XkmK1C1zH-u8hy04No-SzsVh5iH6csKkYsEtyBzb84E,3479
|
|
63
63
|
port_ocean/clients/port/mixins/blueprints.py,sha256=iAKwguhDpUL-YLd7GRNjS-monVgOG8UyKJFOengO_zM,4291
|
|
64
|
-
port_ocean/clients/port/mixins/entities.py,sha256=
|
|
64
|
+
port_ocean/clients/port/mixins/entities.py,sha256=S3axuNeff2sIX1Xq6AAVuyW1kPhM12thkLX5AywitYs,26106
|
|
65
65
|
port_ocean/clients/port/mixins/integrations.py,sha256=rzmfv3BfsBXX21VZrhZLsH5B5spvVBo6xIiXKxOwNvg,12236
|
|
66
66
|
port_ocean/clients/port/mixins/migrations.py,sha256=vdL_A_NNUogvzujyaRLIoZEu5vmKDY2BxTjoGP94YzI,1467
|
|
67
67
|
port_ocean/clients/port/mixins/organization.py,sha256=A2cP5V49KnjoAXxjmnm_XGth4ftPSU0qURNfnyUyS_Y,1041
|
|
@@ -221,8 +221,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
|
|
|
221
221
|
port_ocean/utils/signal.py,sha256=J1sI-e_32VHP_VUa5bskLMFoJjJOAk5isrnewKDikUI,2125
|
|
222
222
|
port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
|
|
223
223
|
port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
|
|
224
|
-
port_ocean-0.30.
|
|
225
|
-
port_ocean-0.30.
|
|
226
|
-
port_ocean-0.30.
|
|
227
|
-
port_ocean-0.30.
|
|
228
|
-
port_ocean-0.30.
|
|
224
|
+
port_ocean-0.30.4.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
|
225
|
+
port_ocean-0.30.4.dist-info/METADATA,sha256=V6JgHyFwlUviArX3z3Wch_wzNtrPi8HZ1prwThOOKzY,7095
|
|
226
|
+
port_ocean-0.30.4.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
227
|
+
port_ocean-0.30.4.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
|
|
228
|
+
port_ocean-0.30.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|