port-ocean 0.21.5__py3-none-any.whl → 0.22.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of port-ocean might be problematic. Click here for more details.

Files changed (35) hide show
  1. integrations/_infra/Makefile +2 -0
  2. port_ocean/cli/cookiecutter/cookiecutter.json +2 -2
  3. port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +1 -1
  4. port_ocean/clients/port/mixins/entities.py +11 -12
  5. port_ocean/config/settings.py +22 -0
  6. port_ocean/context/event.py +1 -0
  7. port_ocean/context/ocean.py +5 -0
  8. port_ocean/context/resource.py +3 -4
  9. port_ocean/core/defaults/common.py +1 -0
  10. port_ocean/core/defaults/initialize.py +1 -1
  11. port_ocean/core/event_listener/base.py +6 -3
  12. port_ocean/core/handlers/entities_state_applier/port/applier.py +23 -21
  13. port_ocean/core/handlers/entity_processor/base.py +0 -2
  14. port_ocean/core/handlers/port_app_config/models.py +1 -1
  15. port_ocean/core/handlers/resync_state_updater/updater.py +9 -0
  16. port_ocean/core/integrations/mixins/sync_raw.py +61 -10
  17. port_ocean/core/models.py +6 -2
  18. port_ocean/core/ocean_types.py +1 -0
  19. port_ocean/core/utils/utils.py +10 -2
  20. port_ocean/helpers/metric/metric.py +238 -0
  21. port_ocean/helpers/metric/utils.py +30 -0
  22. port_ocean/helpers/retry.py +2 -1
  23. port_ocean/ocean.py +17 -4
  24. port_ocean/tests/clients/port/mixins/test_entities.py +12 -9
  25. port_ocean/tests/core/conftest.py +187 -0
  26. port_ocean/tests/core/handlers/entities_state_applier/test_applier.py +154 -6
  27. port_ocean/tests/core/handlers/mixins/test_sync_raw.py +29 -164
  28. port_ocean/tests/core/utils/test_resolve_entities_diff.py +52 -0
  29. port_ocean/tests/test_metric.py +180 -0
  30. port_ocean/utils/async_http.py +4 -1
  31. {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/METADATA +2 -1
  32. {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/RECORD +35 -31
  33. {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/LICENSE.md +0 -0
  34. {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/WHEEL +0 -0
  35. {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/entry_points.txt +0 -0
@@ -11,6 +11,8 @@ define run_checks
11
11
  ruff check . || exit_code=$$?; \
12
12
  echo "Running black"; \
13
13
  black --check . || exit_code=$$?; \
14
+ echo "Running yamllint"; \
15
+ yamllint . || exit_code=$$?; \
14
16
  if [ $$exit_code -eq 1 ]; then \
15
17
  echo "\033[0;31mOne or more checks failed with exit code $$exit_code\033[0m"; \
16
18
  else \
@@ -6,8 +6,8 @@
6
6
  "email": "Your address email <you@example.com>",
7
7
  "release_date": "{% now 'local' %}",
8
8
  "is_private_integration": true,
9
- "port_client_id": "you can find it using: https://docs.getport.io/build-your-software-catalog/custom-integration/api/#find-your-port-credentials",
10
- "port_client_secret": "you can find it using: https://docs.getport.io/build-your-software-catalog/custom-integration/api/#find-your-port-credentials",
9
+ "port_client_id": "you can find it using: https://docs.port.io/build-your-software-catalog/custom-integration/api/#find-your-port-credentials",
10
+ "port_client_secret": "you can find it using: https://docs.port.io/build-your-software-catalog/custom-integration/api/#find-your-port-credentials",
11
11
  "is_us_region": false,
12
12
  "_extensions": [
13
13
  "jinja2_time.TimeExtension",
@@ -2,6 +2,6 @@
2
2
 
3
3
  An integration used to import {{cookiecutter.integration_name}} resources into Port.
4
4
 
5
- #### Install & use the integration - [Integration documentation](https://docs.getport.io/build-your-software-catalog/sync-data-to-catalog/) *Replace this link with a link to this integration's documentation*
5
+ #### Install & use the integration - [Integration documentation](https://docs.port.io/build-your-software-catalog/sync-data-to-catalog/) *Replace this link with a link to this integration's documentation*
6
6
 
7
7
  #### Develop & improve the integration - [Ocean integration development documentation](https://ocean.getport.io/develop-an-integration/)
@@ -2,6 +2,7 @@ import asyncio
2
2
  from typing import Any, Literal
3
3
  from urllib.parse import quote_plus
4
4
 
5
+
5
6
  import httpx
6
7
  from loguru import logger
7
8
 
@@ -99,7 +100,6 @@ class EntityClientMixin:
99
100
  # We return None to ignore the entity later in the delete process
100
101
  if result_entity.is_using_search_identifier:
101
102
  return None
102
-
103
103
  return self._reduce_entity(result_entity)
104
104
 
105
105
  @staticmethod
@@ -125,7 +125,6 @@ class EntityClientMixin:
125
125
  key: None if isinstance(relation, dict) else relation
126
126
  for key, relation in entity.relations.items()
127
127
  }
128
-
129
128
  return reduced_entity
130
129
 
131
130
  async def batch_upsert_entities(
@@ -134,7 +133,7 @@ class EntityClientMixin:
134
133
  request_options: RequestOptions,
135
134
  user_agent_type: UserAgentType | None = None,
136
135
  should_raise: bool = True,
137
- ) -> list[Entity]:
136
+ ) -> list[tuple[bool, Entity]]:
138
137
  modified_entities_results = await asyncio.gather(
139
138
  *(
140
139
  self.upsert_entity(
@@ -147,17 +146,17 @@ class EntityClientMixin:
147
146
  ),
148
147
  return_exceptions=True,
149
148
  )
150
- entity_results = [
151
- entity for entity in modified_entities_results if isinstance(entity, Entity)
152
- ]
153
- if not should_raise:
154
- return entity_results
155
149
 
156
- for entity_result in modified_entities_results:
157
- if isinstance(entity_result, Exception):
158
- raise entity_result
150
+ entities_results: list[tuple[bool, Entity]] = []
151
+ for original_entity, result in zip(entities, modified_entities_results):
152
+ if isinstance(result, Exception) and should_raise:
153
+ raise result
154
+ elif isinstance(result, Entity):
155
+ entities_results.append((True, result))
156
+ elif result is False:
157
+ entities_results.append((False, original_entity))
159
158
 
160
- return entity_results
159
+ return entities_results
161
160
 
162
161
  async def delete_entity(
163
162
  self,
@@ -61,6 +61,11 @@ class IntegrationSettings(BaseOceanModel, extra=Extra.allow):
61
61
  return values
62
62
 
63
63
 
64
+ class MetricsSettings(BaseOceanModel, extra=Extra.allow):
65
+ enabled: bool = Field(default=False)
66
+ webhook_url: str | None = Field(default=None)
67
+
68
+
64
69
  class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
65
70
  _integration_config_model: BaseModel | None = None
66
71
 
@@ -83,9 +88,26 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
83
88
  )
84
89
  runtime: Runtime = Runtime.OnPrem
85
90
  resources_path: str = Field(default=".port/resources")
91
+ metrics: MetricsSettings = Field(
92
+ default_factory=lambda: MetricsSettings(enabled=False, webhook_url=None)
93
+ )
86
94
  max_event_processing_seconds: float = 90.0
87
95
  max_wait_seconds_before_shutdown: float = 5.0
88
96
 
97
+ @validator("metrics", pre=True)
98
+ def validate_metrics(cls, v: Any) -> MetricsSettings | dict[str, Any] | None:
99
+ if v is None:
100
+ return MetricsSettings(enabled=False, webhook_url=None)
101
+ if isinstance(v, dict):
102
+ return v
103
+ if isinstance(v, MetricsSettings):
104
+ return v
105
+ # Try to convert to dict for other types
106
+ try:
107
+ return dict(v)
108
+ except (TypeError, ValueError):
109
+ return MetricsSettings(enabled=False, webhook_url=None)
110
+
89
111
  @root_validator()
90
112
  def validate_integration_config(cls, values: dict[str, Any]) -> dict[str, Any]:
91
113
  if not (config_model := values.get("_integration_config_model")):
@@ -142,6 +142,7 @@ async def event_context(
142
142
  )
143
143
 
144
144
  attributes = {**parent_attributes, **(attributes or {})}
145
+
145
146
  new_event = EventContext(
146
147
  event_type,
147
148
  trigger_type=trigger_type,
@@ -1,6 +1,7 @@
1
1
  from typing import Callable, TYPE_CHECKING, Any, Literal, Union
2
2
 
3
3
  from fastapi import APIRouter
4
+ from port_ocean.helpers.metric.metric import Metrics
4
5
  from pydantic.main import BaseModel
5
6
  from werkzeug.local import LocalProxy
6
7
 
@@ -41,6 +42,10 @@ class PortOceanContext:
41
42
  )
42
43
  return self._app
43
44
 
45
+ @property
46
+ def metrics(self) -> Metrics:
47
+ return self.app.metrics
48
+
44
49
  @property
45
50
  def initialized(self) -> bool:
46
51
  return self._app is not None
@@ -23,6 +23,7 @@ class ResourceContext:
23
23
  """
24
24
 
25
25
  resource_config: "ResourceConfig"
26
+ index: int
26
27
 
27
28
  @property
28
29
  def kind(self) -> str:
@@ -50,12 +51,10 @@ resource: ResourceContext = LocalProxy(lambda: _get_resource_context()) # type:
50
51
 
51
52
  @asynccontextmanager
52
53
  async def resource_context(
53
- resource_config: "ResourceConfig",
54
+ resource_config: "ResourceConfig", index: int = 0
54
55
  ) -> AsyncIterator[ResourceContext]:
55
56
  _resource_context_stack.push(
56
- ResourceContext(
57
- resource_config=resource_config,
58
- )
57
+ ResourceContext(resource_config=resource_config, index=index)
59
58
  )
60
59
 
61
60
  with logger.contextualize(resource_kind=resource.kind):
@@ -68,6 +68,7 @@ def deconstruct_blueprints_to_creation_steps(
68
68
  with_relations.append(blueprint.copy())
69
69
 
70
70
  blueprint.pop("teamInheritance", {})
71
+ blueprint.pop("ownership", {})
71
72
  blueprint.pop("relations", {})
72
73
  bare_blueprint.append(blueprint)
73
74
 
@@ -42,8 +42,8 @@ def deconstruct_blueprints_to_creation_steps(
42
42
  blueprint.pop("mirrorProperties", {})
43
43
  blueprint.pop("aggregationProperties", {})
44
44
  with_relations.append(blueprint.copy())
45
-
46
45
  blueprint.pop("teamInheritance", {})
46
+ blueprint.pop("ownership", {})
47
47
  blueprint.pop("relations", {})
48
48
  bare_blueprint.append(blueprint)
49
49
 
@@ -14,7 +14,7 @@ class EventListenerEvents(TypedDict):
14
14
  A dictionary containing event types and their corresponding event handlers.
15
15
  """
16
16
 
17
- on_resync: Callable[[dict[Any, Any]], Awaitable[None]]
17
+ on_resync: Callable[[dict[Any, Any]], Awaitable[bool]]
18
18
 
19
19
 
20
20
  class BaseEventListener:
@@ -67,8 +67,11 @@ class BaseEventListener:
67
67
  """
68
68
  await self._before_resync()
69
69
  try:
70
- await self.events["on_resync"](resync_args)
71
- await self._after_resync()
70
+ resync_succeeded = await self.events["on_resync"](resync_args)
71
+ if resync_succeeded:
72
+ await self._after_resync()
73
+ else:
74
+ await self._on_resync_failure(Exception("Resync failed"))
72
75
  except Exception as e:
73
76
  await self._on_resync_failure(e)
74
77
  raise e
@@ -8,7 +8,9 @@ from port_ocean.core.handlers.entities_state_applier.base import (
8
8
  from port_ocean.core.handlers.entities_state_applier.port.get_related_entities import (
9
9
  get_related_entities,
10
10
  )
11
-
11
+ from port_ocean.context.ocean import ocean
12
+ from port_ocean.helpers.metric.metric import MetricType, MetricPhase
13
+ from port_ocean.helpers.metric.utils import TimeMetric
12
14
  from port_ocean.core.models import Entity
13
15
  from port_ocean.core.ocean_types import EntityDiff
14
16
  from port_ocean.core.utils.entity_topological_sorter import EntityTopologicalSorter
@@ -23,6 +25,7 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
23
25
  through HTTP requests.
24
26
  """
25
27
 
28
+ @TimeMetric(MetricPhase.DELETE)
26
29
  async def _safe_delete(
27
30
  self,
28
31
  entities_to_delete: list[Entity],
@@ -100,6 +103,11 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
100
103
  and deletion_rate <= entity_deletion_threshold
101
104
  ):
102
105
  await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
106
+ ocean.metrics.set_metric(
107
+ name=MetricType.DELETION_COUNT_NAME,
108
+ labels=[ocean.metrics.current_resource_kind(), MetricPhase.DELETE],
109
+ value=len(diff.deleted),
110
+ )
103
111
  else:
104
112
  logger.info(
105
113
  f"Skipping deletion of entities with deletion rate {deletion_rate}",
@@ -113,26 +121,20 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
113
121
  ) -> list[Entity]:
114
122
  logger.info(f"Upserting {len(entities)} entities")
115
123
  modified_entities: list[Entity] = []
116
- if event.port_app_config.create_missing_related_entities:
117
- modified_entities = await self.context.port_client.batch_upsert_entities(
118
- entities,
119
- event.port_app_config.get_port_request_options(),
120
- user_agent_type,
121
- should_raise=False,
122
- )
123
- else:
124
- for entity in entities:
125
- upsertedEntity = await self.context.port_client.upsert_entity(
126
- entity,
127
- event.port_app_config.get_port_request_options(),
128
- user_agent_type,
129
- should_raise=False,
130
- )
131
- if upsertedEntity:
132
- modified_entities.append(upsertedEntity)
133
- # condition to false to differentiate from `result_entity.is_using_search_identifier`
134
- if upsertedEntity is False:
135
- event.entity_topological_sorter.register_entity(entity)
124
+ upserted_entities: list[tuple[bool, Entity]] = []
125
+
126
+ upserted_entities = await self.context.port_client.batch_upsert_entities(
127
+ entities,
128
+ event.port_app_config.get_port_request_options(),
129
+ user_agent_type,
130
+ should_raise=False,
131
+ )
132
+
133
+ for is_upserted, entity in upserted_entities:
134
+ if is_upserted:
135
+ modified_entities.append(entity)
136
+ else:
137
+ event.entity_topological_sorter.register_entity(entity)
136
138
  return modified_entities
137
139
 
138
140
  async def delete(
@@ -1,7 +1,6 @@
1
1
  from abc import abstractmethod
2
2
 
3
3
  from loguru import logger
4
-
5
4
  from port_ocean.core.handlers.base import BaseHandler
6
5
  from port_ocean.core.handlers.port_app_config.models import ResourceConfig
7
6
  from port_ocean.core.ocean_types import (
@@ -51,7 +50,6 @@ class BaseEntityProcessor(BaseHandler):
51
50
  with logger.contextualize(kind=mapping.kind):
52
51
  if not raw_data:
53
52
  return CalculationResult(EntitySelectorDiff([], []), [])
54
-
55
53
  return await self._parse_items(
56
54
  mapping, raw_data, parse_all, send_raw_data_examples_amount
57
55
  )
@@ -22,7 +22,7 @@ class EntityMapping(BaseModel):
22
22
  identifier: str | IngestSearchQuery
23
23
  title: str | None
24
24
  blueprint: str
25
- team: str | None
25
+ team: str | IngestSearchQuery | None
26
26
  properties: dict[str, str] = Field(default_factory=dict)
27
27
  relations: dict[str, str | IngestSearchQuery] = Field(default_factory=dict)
28
28
 
@@ -3,6 +3,8 @@ from typing import Any, Literal
3
3
  from port_ocean.clients.port.client import PortClient
4
4
  from port_ocean.utils.misc import IntegrationStateStatus
5
5
  from port_ocean.utils.time import get_next_occurrence
6
+ from port_ocean.context.ocean import ocean
7
+ from port_ocean.helpers.metric.metric import MetricType, MetricPhase
6
8
 
7
9
 
8
10
  class ResyncStateUpdater:
@@ -82,3 +84,10 @@ class ResyncStateUpdater:
82
84
  self.last_integration_state_updated_at = integration["resyncState"][
83
85
  "updatedAt"
84
86
  ]
87
+
88
+ ocean.metrics.set_metric(
89
+ name=MetricType.SUCCESS_NAME,
90
+ labels=[ocean.metrics.current_resource_kind(), MetricPhase.RESYNC],
91
+ value=int(status == IntegrationStateStatus.Completed),
92
+ )
93
+ await ocean.metrics.flush(kind=ocean.metrics.current_resource_kind())
@@ -11,6 +11,7 @@ from port_ocean.clients.port.types import UserAgentType
11
11
  from port_ocean.context.event import TriggerType, event_context, EventType, event
12
12
  from port_ocean.context.ocean import ocean
13
13
  from port_ocean.context.resource import resource_context
14
+ from port_ocean.context import resource
14
15
  from port_ocean.core.handlers.port_app_config.models import ResourceConfig
15
16
  from port_ocean.core.integrations.mixins import HandlerMixin, EventsMixin
16
17
  from port_ocean.core.integrations.mixins.utils import (
@@ -30,6 +31,8 @@ from port_ocean.core.ocean_types import (
30
31
  )
31
32
  from port_ocean.core.utils.utils import resolve_entities_diff, zip_and_sum, gather_and_split_errors_from_results
32
33
  from port_ocean.exceptions.core import OceanAbortException
34
+ from port_ocean.helpers.metric.metric import MetricType, MetricPhase
35
+ from port_ocean.helpers.metric.utils import TimeMetric
33
36
 
34
37
  SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
35
38
 
@@ -190,6 +193,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
190
193
  return resolve_entities_diff(entities, entities_at_port_with_properties)
191
194
  return entities
192
195
 
196
+
193
197
  async def _fetch_entities_batch_from_port(
194
198
  self,
195
199
  entities_batch: list[Entity],
@@ -252,7 +256,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
252
256
 
253
257
 
254
258
  return CalculationResult(
255
- objects_diff[0].entity_selector_diff._replace(passed=modified_objects),
259
+ number_of_transformed_entities=len(objects_diff[0].entity_selector_diff.passed),
260
+ entity_selector_diff=objects_diff[0].entity_selector_diff._replace(passed=modified_objects),
256
261
  errors=objects_diff[0].errors,
257
262
  misonfigured_entity_keys=objects_diff[0].misonfigured_entity_keys
258
263
  )
@@ -270,7 +275,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
270
275
  return [], []
271
276
 
272
277
  objects_diff = await self._calculate_raw([(resource, results)])
273
- entities_selector_diff, errors, _ = objects_diff[0]
278
+ entities_selector_diff, errors, _, _ = objects_diff[0]
274
279
 
275
280
  await self.entities_state_applier.delete(
276
281
  entities_selector_diff.passed, user_agent_type
@@ -278,6 +283,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
278
283
  logger.info("Finished unregistering change")
279
284
  return entities_selector_diff.passed, errors
280
285
 
286
+ @TimeMetric(MetricPhase.RESYNC)
281
287
  async def _register_in_batches(
282
288
  self, resource_config: ResourceConfig, user_agent_type: UserAgentType
283
289
  ) -> tuple[list[Entity], list[Exception]]:
@@ -304,10 +310,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
304
310
  )
305
311
  errors.extend(calculation_result.errors)
306
312
  passed_entities = list(calculation_result.entity_selector_diff.passed)
313
+ logger.info(
314
+ f"Finished registering change for {len(raw_results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
315
+ )
307
316
 
317
+ number_of_raw_results = 0
318
+ number_of_transformed_entities = 0
308
319
  for generator in async_generators:
309
320
  try:
310
321
  async for items in generator:
322
+ number_of_raw_results += len(items)
311
323
  if send_raw_data_examples_amount > 0:
312
324
  send_raw_data_examples_amount = max(
313
325
  0, send_raw_data_examples_amount - len(passed_entities)
@@ -321,12 +333,32 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
321
333
  )
322
334
  errors.extend(calculation_result.errors)
323
335
  passed_entities.extend(calculation_result.entity_selector_diff.passed)
336
+ number_of_transformed_entities += calculation_result.number_of_transformed_entities
324
337
  except* OceanAbortException as error:
325
338
  errors.append(error)
326
339
 
327
340
  logger.info(
328
- f"Finished registering change for {len(results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
341
+ f"Finished registering kind: {resource_config.kind}-{resource.resource.index} ,{len(passed_entities)} entities out of {number_of_raw_results} raw results"
342
+ )
343
+
344
+ ocean.metrics.set_metric(
345
+ name=MetricType.SUCCESS_NAME,
346
+ labels=[ocean.metrics.current_resource_kind(), MetricPhase.RESYNC],
347
+ value=int(not errors)
329
348
  )
349
+
350
+ ocean.metrics.set_metric(
351
+ name=MetricType.OBJECT_COUNT_NAME,
352
+ labels=[ocean.metrics.current_resource_kind(), MetricPhase.EXTRACT],
353
+ value=number_of_raw_results
354
+ )
355
+
356
+ ocean.metrics.set_metric(
357
+ name=MetricType.OBJECT_COUNT_NAME,
358
+ labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM],
359
+ value=number_of_transformed_entities
360
+ )
361
+
330
362
  return passed_entities, errors
331
363
 
332
364
  async def register_raw(
@@ -356,7 +388,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
356
388
  if not resource_mappings:
357
389
  return []
358
390
 
359
- diffs, errors, misconfigured_entity_keys = zip(
391
+ diffs, errors, _, misconfigured_entity_keys = zip(
360
392
  *await asyncio.gather(
361
393
  *(
362
394
  self._register_resource_raw(
@@ -510,6 +542,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
510
542
  {"before": entities_before_flatten, "after": entities_after_flatten},
511
543
  user_agent_type,
512
544
  )
545
+
513
546
  async def sort_and_upsert_failed_entities(self,user_agent_type: UserAgentType)->None:
514
547
  try:
515
548
  if not event.entity_topological_sorter.should_execute():
@@ -524,13 +557,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
524
557
  if isinstance(ocean_abort.__cause__,CycleError):
525
558
  for entity in event.entity_topological_sorter.get_entities(False):
526
559
  await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
560
+
561
+
562
+ @TimeMetric(MetricPhase.RESYNC)
527
563
  async def sync_raw_all(
528
564
  self,
529
565
  _: dict[Any, Any] | None = None,
530
566
  trigger_type: TriggerType = "machine",
531
567
  user_agent_type: UserAgentType = UserAgentType.exporter,
532
568
  silent: bool = True,
533
- ) -> None:
569
+ ) -> bool:
534
570
  """Perform a full synchronization of raw entities.
535
571
 
536
572
  This method performs a full synchronization of raw entities, including registration, unregistration,
@@ -543,6 +579,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
543
579
  silent (bool): Whether to raise exceptions or handle them silently.
544
580
  """
545
581
  logger.info("Resync was triggered")
582
+
546
583
  async with event_context(
547
584
  EventType.RESYNC,
548
585
  trigger_type=trigger_type,
@@ -574,18 +611,29 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
574
611
 
575
612
 
576
613
  try:
577
- for resource in app_config.resources:
614
+ for index,resource in enumerate(app_config.resources):
578
615
  # create resource context per resource kind, so resync method could have access to the resource
579
616
  # config as we might have multiple resources in the same event
580
- async with resource_context(resource):
581
- task = asyncio.get_event_loop().create_task(
617
+ async with resource_context(resource,index):
618
+ resource_kind_id = f"{resource.kind}-{index}"
619
+ task = asyncio.create_task(
582
620
  self._register_in_batches(resource, user_agent_type)
583
621
  )
584
622
 
585
623
  event.on_abort(lambda: task.cancel())
586
- creation_results.append(await task)
624
+ kind_results: tuple[list[Entity], list[Exception]] = await task
625
+ ocean.metrics.set_metric(
626
+ name=MetricType.OBJECT_COUNT_NAME,
627
+ labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD],
628
+ value=len(kind_results[0])
629
+ )
630
+
631
+ creation_results.append(kind_results)
632
+
633
+ await ocean.metrics.flush(kind=resource_kind_id)
587
634
 
588
635
  await self.sort_and_upsert_failed_entities(user_agent_type)
636
+
589
637
  except asyncio.CancelledError as e:
590
638
  logger.warning("Resync aborted successfully, skipping delete phase. This leads to an incomplete state")
591
639
  raise
@@ -604,7 +652,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
604
652
  ]
605
653
 
606
654
  if errors:
607
- message = f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state"
655
+ message = f"Resync failed with {len(errors)} errors, skipping delete phase due to incomplete state"
608
656
  error_group = ExceptionGroup(
609
657
  message,
610
658
  errors,
@@ -613,6 +661,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
613
661
  raise error_group
614
662
 
615
663
  logger.error(message, exc_info=error_group)
664
+ return False
616
665
  else:
617
666
  logger.info(
618
667
  f"Running resync diff calculation, number of entities created during sync: {len(generated_entities)}"
@@ -635,3 +684,5 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
635
684
  await resync_complete_fn()
636
685
 
637
686
  logger.info("Finished executing resync_complete hooks")
687
+
688
+ return True
port_ocean/core/models.py CHANGED
@@ -40,7 +40,7 @@ class Entity(BaseModel):
40
40
  identifier: Any
41
41
  blueprint: Any
42
42
  title: Any
43
- team: str | None | list[Any] = []
43
+ team: str | None | list[Any] | dict[str, Any] = []
44
44
  properties: dict[str, Any] = {}
45
45
  relations: dict[str, Any] = {}
46
46
 
@@ -50,7 +50,11 @@ class Entity(BaseModel):
50
50
 
51
51
  @property
52
52
  def is_using_search_relation(self) -> bool:
53
- return any(isinstance(relation, dict) for relation in self.relations.values())
53
+ return any(
54
+ isinstance(relation, dict) for relation in self.relations.values()
55
+ ) or (
56
+ self.team is not None and any(isinstance(team, dict) for team in self.team)
57
+ )
54
58
 
55
59
 
56
60
  class BlueprintRelation(BaseModel):
@@ -41,6 +41,7 @@ class EntitySelectorDiff(NamedTuple):
41
41
  class CalculationResult(NamedTuple):
42
42
  entity_selector_diff: EntitySelectorDiff
43
43
  errors: list[Exception]
44
+ number_of_transformed_entities: int = 0
44
45
  misonfigured_entity_keys: dict[str, str] = field(default_factory=dict)
45
46
 
46
47
 
@@ -110,7 +110,8 @@ def get_port_diff(before: Iterable[Entity], after: Iterable[Entity]) -> EntityPo
110
110
 
111
111
 
112
112
  def are_teams_different(
113
- first_team: str | None | list[Any], second_team: str | None | list[Any]
113
+ first_team: str | None | list[Any] | dict[str, Any],
114
+ second_team: str | None | list[Any] | dict[str, Any],
114
115
  ) -> bool:
115
116
  if isinstance(first_team, list) and isinstance(second_team, list):
116
117
  return sorted(first_team) != sorted(second_team)
@@ -122,6 +123,7 @@ def are_entities_fields_equal(
122
123
  ) -> bool:
123
124
  """
124
125
  Compare two entity fields by serializing them to JSON and comparing their SHA-256 hashes.
126
+ Removes keys with None values before comparison if the corresponding key doesn't exist in the other dict.
125
127
 
126
128
  Args:
127
129
  first_entity_field: First entity field dictionary to compare
@@ -130,7 +132,13 @@ def are_entities_fields_equal(
130
132
  Returns:
131
133
  bool: True if the entity fields have identical content
132
134
  """
133
- first_props = json.dumps(first_entity_field, sort_keys=True)
135
+ first_entity_field_copy = first_entity_field.copy()
136
+
137
+ for key in list(first_entity_field.keys()):
138
+ if first_entity_field[key] is None and key not in second_entity_field:
139
+ del first_entity_field_copy[key]
140
+
141
+ first_props = json.dumps(first_entity_field_copy, sort_keys=True)
134
142
  second_props = json.dumps(second_entity_field, sort_keys=True)
135
143
  first_hash = hashlib.sha256(first_props.encode()).hexdigest()
136
144
  second_hash = hashlib.sha256(second_props.encode()).hexdigest()