port-ocean 0.21.5__py3-none-any.whl → 0.22.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of port-ocean might be problematic. Click here for more details.
- integrations/_infra/Makefile +2 -0
- port_ocean/cli/cookiecutter/cookiecutter.json +2 -2
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +1 -1
- port_ocean/clients/port/mixins/entities.py +11 -12
- port_ocean/config/settings.py +22 -0
- port_ocean/context/event.py +1 -0
- port_ocean/context/ocean.py +5 -0
- port_ocean/context/resource.py +3 -4
- port_ocean/core/defaults/common.py +1 -0
- port_ocean/core/defaults/initialize.py +1 -1
- port_ocean/core/event_listener/base.py +6 -3
- port_ocean/core/handlers/entities_state_applier/port/applier.py +23 -21
- port_ocean/core/handlers/entity_processor/base.py +0 -2
- port_ocean/core/handlers/port_app_config/models.py +1 -1
- port_ocean/core/handlers/resync_state_updater/updater.py +9 -0
- port_ocean/core/integrations/mixins/sync_raw.py +61 -10
- port_ocean/core/models.py +6 -2
- port_ocean/core/ocean_types.py +1 -0
- port_ocean/core/utils/utils.py +10 -2
- port_ocean/helpers/metric/metric.py +238 -0
- port_ocean/helpers/metric/utils.py +30 -0
- port_ocean/helpers/retry.py +2 -1
- port_ocean/ocean.py +17 -4
- port_ocean/tests/clients/port/mixins/test_entities.py +12 -9
- port_ocean/tests/core/conftest.py +187 -0
- port_ocean/tests/core/handlers/entities_state_applier/test_applier.py +154 -6
- port_ocean/tests/core/handlers/mixins/test_sync_raw.py +29 -164
- port_ocean/tests/core/utils/test_resolve_entities_diff.py +52 -0
- port_ocean/tests/test_metric.py +180 -0
- port_ocean/utils/async_http.py +4 -1
- {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/METADATA +2 -1
- {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/RECORD +35 -31
- {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/LICENSE.md +0 -0
- {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/WHEEL +0 -0
- {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/entry_points.txt +0 -0
integrations/_infra/Makefile
CHANGED
|
@@ -11,6 +11,8 @@ define run_checks
|
|
|
11
11
|
ruff check . || exit_code=$$?; \
|
|
12
12
|
echo "Running black"; \
|
|
13
13
|
black --check . || exit_code=$$?; \
|
|
14
|
+
echo "Running yamllint"; \
|
|
15
|
+
yamllint . || exit_code=$$?; \
|
|
14
16
|
if [ $$exit_code -eq 1 ]; then \
|
|
15
17
|
echo "\033[0;31mOne or more checks failed with exit code $$exit_code\033[0m"; \
|
|
16
18
|
else \
|
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
"email": "Your address email <you@example.com>",
|
|
7
7
|
"release_date": "{% now 'local' %}",
|
|
8
8
|
"is_private_integration": true,
|
|
9
|
-
"port_client_id": "you can find it using: https://docs.
|
|
10
|
-
"port_client_secret": "you can find it using: https://docs.
|
|
9
|
+
"port_client_id": "you can find it using: https://docs.port.io/build-your-software-catalog/custom-integration/api/#find-your-port-credentials",
|
|
10
|
+
"port_client_secret": "you can find it using: https://docs.port.io/build-your-software-catalog/custom-integration/api/#find-your-port-credentials",
|
|
11
11
|
"is_us_region": false,
|
|
12
12
|
"_extensions": [
|
|
13
13
|
"jinja2_time.TimeExtension",
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
An integration used to import {{cookiecutter.integration_name}} resources into Port.
|
|
4
4
|
|
|
5
|
-
#### Install & use the integration - [Integration documentation](https://docs.
|
|
5
|
+
#### Install & use the integration - [Integration documentation](https://docs.port.io/build-your-software-catalog/sync-data-to-catalog/) *Replace this link with a link to this integration's documentation*
|
|
6
6
|
|
|
7
7
|
#### Develop & improve the integration - [Ocean integration development documentation](https://ocean.getport.io/develop-an-integration/)
|
|
@@ -2,6 +2,7 @@ import asyncio
|
|
|
2
2
|
from typing import Any, Literal
|
|
3
3
|
from urllib.parse import quote_plus
|
|
4
4
|
|
|
5
|
+
|
|
5
6
|
import httpx
|
|
6
7
|
from loguru import logger
|
|
7
8
|
|
|
@@ -99,7 +100,6 @@ class EntityClientMixin:
|
|
|
99
100
|
# We return None to ignore the entity later in the delete process
|
|
100
101
|
if result_entity.is_using_search_identifier:
|
|
101
102
|
return None
|
|
102
|
-
|
|
103
103
|
return self._reduce_entity(result_entity)
|
|
104
104
|
|
|
105
105
|
@staticmethod
|
|
@@ -125,7 +125,6 @@ class EntityClientMixin:
|
|
|
125
125
|
key: None if isinstance(relation, dict) else relation
|
|
126
126
|
for key, relation in entity.relations.items()
|
|
127
127
|
}
|
|
128
|
-
|
|
129
128
|
return reduced_entity
|
|
130
129
|
|
|
131
130
|
async def batch_upsert_entities(
|
|
@@ -134,7 +133,7 @@ class EntityClientMixin:
|
|
|
134
133
|
request_options: RequestOptions,
|
|
135
134
|
user_agent_type: UserAgentType | None = None,
|
|
136
135
|
should_raise: bool = True,
|
|
137
|
-
) -> list[Entity]:
|
|
136
|
+
) -> list[tuple[bool, Entity]]:
|
|
138
137
|
modified_entities_results = await asyncio.gather(
|
|
139
138
|
*(
|
|
140
139
|
self.upsert_entity(
|
|
@@ -147,17 +146,17 @@ class EntityClientMixin:
|
|
|
147
146
|
),
|
|
148
147
|
return_exceptions=True,
|
|
149
148
|
)
|
|
150
|
-
entity_results = [
|
|
151
|
-
entity for entity in modified_entities_results if isinstance(entity, Entity)
|
|
152
|
-
]
|
|
153
|
-
if not should_raise:
|
|
154
|
-
return entity_results
|
|
155
149
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
150
|
+
entities_results: list[tuple[bool, Entity]] = []
|
|
151
|
+
for original_entity, result in zip(entities, modified_entities_results):
|
|
152
|
+
if isinstance(result, Exception) and should_raise:
|
|
153
|
+
raise result
|
|
154
|
+
elif isinstance(result, Entity):
|
|
155
|
+
entities_results.append((True, result))
|
|
156
|
+
elif result is False:
|
|
157
|
+
entities_results.append((False, original_entity))
|
|
159
158
|
|
|
160
|
-
return
|
|
159
|
+
return entities_results
|
|
161
160
|
|
|
162
161
|
async def delete_entity(
|
|
163
162
|
self,
|
port_ocean/config/settings.py
CHANGED
|
@@ -61,6 +61,11 @@ class IntegrationSettings(BaseOceanModel, extra=Extra.allow):
|
|
|
61
61
|
return values
|
|
62
62
|
|
|
63
63
|
|
|
64
|
+
class MetricsSettings(BaseOceanModel, extra=Extra.allow):
|
|
65
|
+
enabled: bool = Field(default=False)
|
|
66
|
+
webhook_url: str | None = Field(default=None)
|
|
67
|
+
|
|
68
|
+
|
|
64
69
|
class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
|
|
65
70
|
_integration_config_model: BaseModel | None = None
|
|
66
71
|
|
|
@@ -83,9 +88,26 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
|
|
|
83
88
|
)
|
|
84
89
|
runtime: Runtime = Runtime.OnPrem
|
|
85
90
|
resources_path: str = Field(default=".port/resources")
|
|
91
|
+
metrics: MetricsSettings = Field(
|
|
92
|
+
default_factory=lambda: MetricsSettings(enabled=False, webhook_url=None)
|
|
93
|
+
)
|
|
86
94
|
max_event_processing_seconds: float = 90.0
|
|
87
95
|
max_wait_seconds_before_shutdown: float = 5.0
|
|
88
96
|
|
|
97
|
+
@validator("metrics", pre=True)
|
|
98
|
+
def validate_metrics(cls, v: Any) -> MetricsSettings | dict[str, Any] | None:
|
|
99
|
+
if v is None:
|
|
100
|
+
return MetricsSettings(enabled=False, webhook_url=None)
|
|
101
|
+
if isinstance(v, dict):
|
|
102
|
+
return v
|
|
103
|
+
if isinstance(v, MetricsSettings):
|
|
104
|
+
return v
|
|
105
|
+
# Try to convert to dict for other types
|
|
106
|
+
try:
|
|
107
|
+
return dict(v)
|
|
108
|
+
except (TypeError, ValueError):
|
|
109
|
+
return MetricsSettings(enabled=False, webhook_url=None)
|
|
110
|
+
|
|
89
111
|
@root_validator()
|
|
90
112
|
def validate_integration_config(cls, values: dict[str, Any]) -> dict[str, Any]:
|
|
91
113
|
if not (config_model := values.get("_integration_config_model")):
|
port_ocean/context/event.py
CHANGED
port_ocean/context/ocean.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from typing import Callable, TYPE_CHECKING, Any, Literal, Union
|
|
2
2
|
|
|
3
3
|
from fastapi import APIRouter
|
|
4
|
+
from port_ocean.helpers.metric.metric import Metrics
|
|
4
5
|
from pydantic.main import BaseModel
|
|
5
6
|
from werkzeug.local import LocalProxy
|
|
6
7
|
|
|
@@ -41,6 +42,10 @@ class PortOceanContext:
|
|
|
41
42
|
)
|
|
42
43
|
return self._app
|
|
43
44
|
|
|
45
|
+
@property
|
|
46
|
+
def metrics(self) -> Metrics:
|
|
47
|
+
return self.app.metrics
|
|
48
|
+
|
|
44
49
|
@property
|
|
45
50
|
def initialized(self) -> bool:
|
|
46
51
|
return self._app is not None
|
port_ocean/context/resource.py
CHANGED
|
@@ -23,6 +23,7 @@ class ResourceContext:
|
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
25
|
resource_config: "ResourceConfig"
|
|
26
|
+
index: int
|
|
26
27
|
|
|
27
28
|
@property
|
|
28
29
|
def kind(self) -> str:
|
|
@@ -50,12 +51,10 @@ resource: ResourceContext = LocalProxy(lambda: _get_resource_context()) # type:
|
|
|
50
51
|
|
|
51
52
|
@asynccontextmanager
|
|
52
53
|
async def resource_context(
|
|
53
|
-
resource_config: "ResourceConfig",
|
|
54
|
+
resource_config: "ResourceConfig", index: int = 0
|
|
54
55
|
) -> AsyncIterator[ResourceContext]:
|
|
55
56
|
_resource_context_stack.push(
|
|
56
|
-
ResourceContext(
|
|
57
|
-
resource_config=resource_config,
|
|
58
|
-
)
|
|
57
|
+
ResourceContext(resource_config=resource_config, index=index)
|
|
59
58
|
)
|
|
60
59
|
|
|
61
60
|
with logger.contextualize(resource_kind=resource.kind):
|
|
@@ -42,8 +42,8 @@ def deconstruct_blueprints_to_creation_steps(
|
|
|
42
42
|
blueprint.pop("mirrorProperties", {})
|
|
43
43
|
blueprint.pop("aggregationProperties", {})
|
|
44
44
|
with_relations.append(blueprint.copy())
|
|
45
|
-
|
|
46
45
|
blueprint.pop("teamInheritance", {})
|
|
46
|
+
blueprint.pop("ownership", {})
|
|
47
47
|
blueprint.pop("relations", {})
|
|
48
48
|
bare_blueprint.append(blueprint)
|
|
49
49
|
|
|
@@ -14,7 +14,7 @@ class EventListenerEvents(TypedDict):
|
|
|
14
14
|
A dictionary containing event types and their corresponding event handlers.
|
|
15
15
|
"""
|
|
16
16
|
|
|
17
|
-
on_resync: Callable[[dict[Any, Any]], Awaitable[
|
|
17
|
+
on_resync: Callable[[dict[Any, Any]], Awaitable[bool]]
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class BaseEventListener:
|
|
@@ -67,8 +67,11 @@ class BaseEventListener:
|
|
|
67
67
|
"""
|
|
68
68
|
await self._before_resync()
|
|
69
69
|
try:
|
|
70
|
-
await self.events["on_resync"](resync_args)
|
|
71
|
-
|
|
70
|
+
resync_succeeded = await self.events["on_resync"](resync_args)
|
|
71
|
+
if resync_succeeded:
|
|
72
|
+
await self._after_resync()
|
|
73
|
+
else:
|
|
74
|
+
await self._on_resync_failure(Exception("Resync failed"))
|
|
72
75
|
except Exception as e:
|
|
73
76
|
await self._on_resync_failure(e)
|
|
74
77
|
raise e
|
|
@@ -8,7 +8,9 @@ from port_ocean.core.handlers.entities_state_applier.base import (
|
|
|
8
8
|
from port_ocean.core.handlers.entities_state_applier.port.get_related_entities import (
|
|
9
9
|
get_related_entities,
|
|
10
10
|
)
|
|
11
|
-
|
|
11
|
+
from port_ocean.context.ocean import ocean
|
|
12
|
+
from port_ocean.helpers.metric.metric import MetricType, MetricPhase
|
|
13
|
+
from port_ocean.helpers.metric.utils import TimeMetric
|
|
12
14
|
from port_ocean.core.models import Entity
|
|
13
15
|
from port_ocean.core.ocean_types import EntityDiff
|
|
14
16
|
from port_ocean.core.utils.entity_topological_sorter import EntityTopologicalSorter
|
|
@@ -23,6 +25,7 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
|
|
|
23
25
|
through HTTP requests.
|
|
24
26
|
"""
|
|
25
27
|
|
|
28
|
+
@TimeMetric(MetricPhase.DELETE)
|
|
26
29
|
async def _safe_delete(
|
|
27
30
|
self,
|
|
28
31
|
entities_to_delete: list[Entity],
|
|
@@ -100,6 +103,11 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
|
|
|
100
103
|
and deletion_rate <= entity_deletion_threshold
|
|
101
104
|
):
|
|
102
105
|
await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
|
|
106
|
+
ocean.metrics.set_metric(
|
|
107
|
+
name=MetricType.DELETION_COUNT_NAME,
|
|
108
|
+
labels=[ocean.metrics.current_resource_kind(), MetricPhase.DELETE],
|
|
109
|
+
value=len(diff.deleted),
|
|
110
|
+
)
|
|
103
111
|
else:
|
|
104
112
|
logger.info(
|
|
105
113
|
f"Skipping deletion of entities with deletion rate {deletion_rate}",
|
|
@@ -113,26 +121,20 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
|
|
|
113
121
|
) -> list[Entity]:
|
|
114
122
|
logger.info(f"Upserting {len(entities)} entities")
|
|
115
123
|
modified_entities: list[Entity] = []
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
)
|
|
131
|
-
if upsertedEntity:
|
|
132
|
-
modified_entities.append(upsertedEntity)
|
|
133
|
-
# condition to false to differentiate from `result_entity.is_using_search_identifier`
|
|
134
|
-
if upsertedEntity is False:
|
|
135
|
-
event.entity_topological_sorter.register_entity(entity)
|
|
124
|
+
upserted_entities: list[tuple[bool, Entity]] = []
|
|
125
|
+
|
|
126
|
+
upserted_entities = await self.context.port_client.batch_upsert_entities(
|
|
127
|
+
entities,
|
|
128
|
+
event.port_app_config.get_port_request_options(),
|
|
129
|
+
user_agent_type,
|
|
130
|
+
should_raise=False,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
for is_upserted, entity in upserted_entities:
|
|
134
|
+
if is_upserted:
|
|
135
|
+
modified_entities.append(entity)
|
|
136
|
+
else:
|
|
137
|
+
event.entity_topological_sorter.register_entity(entity)
|
|
136
138
|
return modified_entities
|
|
137
139
|
|
|
138
140
|
async def delete(
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
from abc import abstractmethod
|
|
2
2
|
|
|
3
3
|
from loguru import logger
|
|
4
|
-
|
|
5
4
|
from port_ocean.core.handlers.base import BaseHandler
|
|
6
5
|
from port_ocean.core.handlers.port_app_config.models import ResourceConfig
|
|
7
6
|
from port_ocean.core.ocean_types import (
|
|
@@ -51,7 +50,6 @@ class BaseEntityProcessor(BaseHandler):
|
|
|
51
50
|
with logger.contextualize(kind=mapping.kind):
|
|
52
51
|
if not raw_data:
|
|
53
52
|
return CalculationResult(EntitySelectorDiff([], []), [])
|
|
54
|
-
|
|
55
53
|
return await self._parse_items(
|
|
56
54
|
mapping, raw_data, parse_all, send_raw_data_examples_amount
|
|
57
55
|
)
|
|
@@ -22,7 +22,7 @@ class EntityMapping(BaseModel):
|
|
|
22
22
|
identifier: str | IngestSearchQuery
|
|
23
23
|
title: str | None
|
|
24
24
|
blueprint: str
|
|
25
|
-
team: str | None
|
|
25
|
+
team: str | IngestSearchQuery | None
|
|
26
26
|
properties: dict[str, str] = Field(default_factory=dict)
|
|
27
27
|
relations: dict[str, str | IngestSearchQuery] = Field(default_factory=dict)
|
|
28
28
|
|
|
@@ -3,6 +3,8 @@ from typing import Any, Literal
|
|
|
3
3
|
from port_ocean.clients.port.client import PortClient
|
|
4
4
|
from port_ocean.utils.misc import IntegrationStateStatus
|
|
5
5
|
from port_ocean.utils.time import get_next_occurrence
|
|
6
|
+
from port_ocean.context.ocean import ocean
|
|
7
|
+
from port_ocean.helpers.metric.metric import MetricType, MetricPhase
|
|
6
8
|
|
|
7
9
|
|
|
8
10
|
class ResyncStateUpdater:
|
|
@@ -82,3 +84,10 @@ class ResyncStateUpdater:
|
|
|
82
84
|
self.last_integration_state_updated_at = integration["resyncState"][
|
|
83
85
|
"updatedAt"
|
|
84
86
|
]
|
|
87
|
+
|
|
88
|
+
ocean.metrics.set_metric(
|
|
89
|
+
name=MetricType.SUCCESS_NAME,
|
|
90
|
+
labels=[ocean.metrics.current_resource_kind(), MetricPhase.RESYNC],
|
|
91
|
+
value=int(status == IntegrationStateStatus.Completed),
|
|
92
|
+
)
|
|
93
|
+
await ocean.metrics.flush(kind=ocean.metrics.current_resource_kind())
|
|
@@ -11,6 +11,7 @@ from port_ocean.clients.port.types import UserAgentType
|
|
|
11
11
|
from port_ocean.context.event import TriggerType, event_context, EventType, event
|
|
12
12
|
from port_ocean.context.ocean import ocean
|
|
13
13
|
from port_ocean.context.resource import resource_context
|
|
14
|
+
from port_ocean.context import resource
|
|
14
15
|
from port_ocean.core.handlers.port_app_config.models import ResourceConfig
|
|
15
16
|
from port_ocean.core.integrations.mixins import HandlerMixin, EventsMixin
|
|
16
17
|
from port_ocean.core.integrations.mixins.utils import (
|
|
@@ -30,6 +31,8 @@ from port_ocean.core.ocean_types import (
|
|
|
30
31
|
)
|
|
31
32
|
from port_ocean.core.utils.utils import resolve_entities_diff, zip_and_sum, gather_and_split_errors_from_results
|
|
32
33
|
from port_ocean.exceptions.core import OceanAbortException
|
|
34
|
+
from port_ocean.helpers.metric.metric import MetricType, MetricPhase
|
|
35
|
+
from port_ocean.helpers.metric.utils import TimeMetric
|
|
33
36
|
|
|
34
37
|
SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
|
|
35
38
|
|
|
@@ -190,6 +193,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
190
193
|
return resolve_entities_diff(entities, entities_at_port_with_properties)
|
|
191
194
|
return entities
|
|
192
195
|
|
|
196
|
+
|
|
193
197
|
async def _fetch_entities_batch_from_port(
|
|
194
198
|
self,
|
|
195
199
|
entities_batch: list[Entity],
|
|
@@ -252,7 +256,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
252
256
|
|
|
253
257
|
|
|
254
258
|
return CalculationResult(
|
|
255
|
-
objects_diff[0].entity_selector_diff.
|
|
259
|
+
number_of_transformed_entities=len(objects_diff[0].entity_selector_diff.passed),
|
|
260
|
+
entity_selector_diff=objects_diff[0].entity_selector_diff._replace(passed=modified_objects),
|
|
256
261
|
errors=objects_diff[0].errors,
|
|
257
262
|
misonfigured_entity_keys=objects_diff[0].misonfigured_entity_keys
|
|
258
263
|
)
|
|
@@ -270,7 +275,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
270
275
|
return [], []
|
|
271
276
|
|
|
272
277
|
objects_diff = await self._calculate_raw([(resource, results)])
|
|
273
|
-
entities_selector_diff, errors, _ = objects_diff[0]
|
|
278
|
+
entities_selector_diff, errors, _, _ = objects_diff[0]
|
|
274
279
|
|
|
275
280
|
await self.entities_state_applier.delete(
|
|
276
281
|
entities_selector_diff.passed, user_agent_type
|
|
@@ -278,6 +283,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
278
283
|
logger.info("Finished unregistering change")
|
|
279
284
|
return entities_selector_diff.passed, errors
|
|
280
285
|
|
|
286
|
+
@TimeMetric(MetricPhase.RESYNC)
|
|
281
287
|
async def _register_in_batches(
|
|
282
288
|
self, resource_config: ResourceConfig, user_agent_type: UserAgentType
|
|
283
289
|
) -> tuple[list[Entity], list[Exception]]:
|
|
@@ -304,10 +310,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
304
310
|
)
|
|
305
311
|
errors.extend(calculation_result.errors)
|
|
306
312
|
passed_entities = list(calculation_result.entity_selector_diff.passed)
|
|
313
|
+
logger.info(
|
|
314
|
+
f"Finished registering change for {len(raw_results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
|
|
315
|
+
)
|
|
307
316
|
|
|
317
|
+
number_of_raw_results = 0
|
|
318
|
+
number_of_transformed_entities = 0
|
|
308
319
|
for generator in async_generators:
|
|
309
320
|
try:
|
|
310
321
|
async for items in generator:
|
|
322
|
+
number_of_raw_results += len(items)
|
|
311
323
|
if send_raw_data_examples_amount > 0:
|
|
312
324
|
send_raw_data_examples_amount = max(
|
|
313
325
|
0, send_raw_data_examples_amount - len(passed_entities)
|
|
@@ -321,12 +333,32 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
321
333
|
)
|
|
322
334
|
errors.extend(calculation_result.errors)
|
|
323
335
|
passed_entities.extend(calculation_result.entity_selector_diff.passed)
|
|
336
|
+
number_of_transformed_entities += calculation_result.number_of_transformed_entities
|
|
324
337
|
except* OceanAbortException as error:
|
|
325
338
|
errors.append(error)
|
|
326
339
|
|
|
327
340
|
logger.info(
|
|
328
|
-
f"Finished registering
|
|
341
|
+
f"Finished registering kind: {resource_config.kind}-{resource.resource.index} ,{len(passed_entities)} entities out of {number_of_raw_results} raw results"
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
ocean.metrics.set_metric(
|
|
345
|
+
name=MetricType.SUCCESS_NAME,
|
|
346
|
+
labels=[ocean.metrics.current_resource_kind(), MetricPhase.RESYNC],
|
|
347
|
+
value=int(not errors)
|
|
329
348
|
)
|
|
349
|
+
|
|
350
|
+
ocean.metrics.set_metric(
|
|
351
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
|
352
|
+
labels=[ocean.metrics.current_resource_kind(), MetricPhase.EXTRACT],
|
|
353
|
+
value=number_of_raw_results
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
ocean.metrics.set_metric(
|
|
357
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
|
358
|
+
labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM],
|
|
359
|
+
value=number_of_transformed_entities
|
|
360
|
+
)
|
|
361
|
+
|
|
330
362
|
return passed_entities, errors
|
|
331
363
|
|
|
332
364
|
async def register_raw(
|
|
@@ -356,7 +388,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
356
388
|
if not resource_mappings:
|
|
357
389
|
return []
|
|
358
390
|
|
|
359
|
-
diffs, errors, misconfigured_entity_keys = zip(
|
|
391
|
+
diffs, errors, _, misconfigured_entity_keys = zip(
|
|
360
392
|
*await asyncio.gather(
|
|
361
393
|
*(
|
|
362
394
|
self._register_resource_raw(
|
|
@@ -510,6 +542,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
510
542
|
{"before": entities_before_flatten, "after": entities_after_flatten},
|
|
511
543
|
user_agent_type,
|
|
512
544
|
)
|
|
545
|
+
|
|
513
546
|
async def sort_and_upsert_failed_entities(self,user_agent_type: UserAgentType)->None:
|
|
514
547
|
try:
|
|
515
548
|
if not event.entity_topological_sorter.should_execute():
|
|
@@ -524,13 +557,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
524
557
|
if isinstance(ocean_abort.__cause__,CycleError):
|
|
525
558
|
for entity in event.entity_topological_sorter.get_entities(False):
|
|
526
559
|
await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
@TimeMetric(MetricPhase.RESYNC)
|
|
527
563
|
async def sync_raw_all(
|
|
528
564
|
self,
|
|
529
565
|
_: dict[Any, Any] | None = None,
|
|
530
566
|
trigger_type: TriggerType = "machine",
|
|
531
567
|
user_agent_type: UserAgentType = UserAgentType.exporter,
|
|
532
568
|
silent: bool = True,
|
|
533
|
-
) ->
|
|
569
|
+
) -> bool:
|
|
534
570
|
"""Perform a full synchronization of raw entities.
|
|
535
571
|
|
|
536
572
|
This method performs a full synchronization of raw entities, including registration, unregistration,
|
|
@@ -543,6 +579,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
543
579
|
silent (bool): Whether to raise exceptions or handle them silently.
|
|
544
580
|
"""
|
|
545
581
|
logger.info("Resync was triggered")
|
|
582
|
+
|
|
546
583
|
async with event_context(
|
|
547
584
|
EventType.RESYNC,
|
|
548
585
|
trigger_type=trigger_type,
|
|
@@ -574,18 +611,29 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
574
611
|
|
|
575
612
|
|
|
576
613
|
try:
|
|
577
|
-
for resource in app_config.resources:
|
|
614
|
+
for index,resource in enumerate(app_config.resources):
|
|
578
615
|
# create resource context per resource kind, so resync method could have access to the resource
|
|
579
616
|
# config as we might have multiple resources in the same event
|
|
580
|
-
async with resource_context(resource):
|
|
581
|
-
|
|
617
|
+
async with resource_context(resource,index):
|
|
618
|
+
resource_kind_id = f"{resource.kind}-{index}"
|
|
619
|
+
task = asyncio.create_task(
|
|
582
620
|
self._register_in_batches(resource, user_agent_type)
|
|
583
621
|
)
|
|
584
622
|
|
|
585
623
|
event.on_abort(lambda: task.cancel())
|
|
586
|
-
|
|
624
|
+
kind_results: tuple[list[Entity], list[Exception]] = await task
|
|
625
|
+
ocean.metrics.set_metric(
|
|
626
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
|
627
|
+
labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD],
|
|
628
|
+
value=len(kind_results[0])
|
|
629
|
+
)
|
|
630
|
+
|
|
631
|
+
creation_results.append(kind_results)
|
|
632
|
+
|
|
633
|
+
await ocean.metrics.flush(kind=resource_kind_id)
|
|
587
634
|
|
|
588
635
|
await self.sort_and_upsert_failed_entities(user_agent_type)
|
|
636
|
+
|
|
589
637
|
except asyncio.CancelledError as e:
|
|
590
638
|
logger.warning("Resync aborted successfully, skipping delete phase. This leads to an incomplete state")
|
|
591
639
|
raise
|
|
@@ -604,7 +652,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
604
652
|
]
|
|
605
653
|
|
|
606
654
|
if errors:
|
|
607
|
-
message = f"Resync failed with {len(errors)}
|
|
655
|
+
message = f"Resync failed with {len(errors)} errors, skipping delete phase due to incomplete state"
|
|
608
656
|
error_group = ExceptionGroup(
|
|
609
657
|
message,
|
|
610
658
|
errors,
|
|
@@ -613,6 +661,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
613
661
|
raise error_group
|
|
614
662
|
|
|
615
663
|
logger.error(message, exc_info=error_group)
|
|
664
|
+
return False
|
|
616
665
|
else:
|
|
617
666
|
logger.info(
|
|
618
667
|
f"Running resync diff calculation, number of entities created during sync: {len(generated_entities)}"
|
|
@@ -635,3 +684,5 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
|
635
684
|
await resync_complete_fn()
|
|
636
685
|
|
|
637
686
|
logger.info("Finished executing resync_complete hooks")
|
|
687
|
+
|
|
688
|
+
return True
|
port_ocean/core/models.py
CHANGED
|
@@ -40,7 +40,7 @@ class Entity(BaseModel):
|
|
|
40
40
|
identifier: Any
|
|
41
41
|
blueprint: Any
|
|
42
42
|
title: Any
|
|
43
|
-
team: str | None | list[Any] = []
|
|
43
|
+
team: str | None | list[Any] | dict[str, Any] = []
|
|
44
44
|
properties: dict[str, Any] = {}
|
|
45
45
|
relations: dict[str, Any] = {}
|
|
46
46
|
|
|
@@ -50,7 +50,11 @@ class Entity(BaseModel):
|
|
|
50
50
|
|
|
51
51
|
@property
|
|
52
52
|
def is_using_search_relation(self) -> bool:
|
|
53
|
-
return any(
|
|
53
|
+
return any(
|
|
54
|
+
isinstance(relation, dict) for relation in self.relations.values()
|
|
55
|
+
) or (
|
|
56
|
+
self.team is not None and any(isinstance(team, dict) for team in self.team)
|
|
57
|
+
)
|
|
54
58
|
|
|
55
59
|
|
|
56
60
|
class BlueprintRelation(BaseModel):
|
port_ocean/core/ocean_types.py
CHANGED
|
@@ -41,6 +41,7 @@ class EntitySelectorDiff(NamedTuple):
|
|
|
41
41
|
class CalculationResult(NamedTuple):
|
|
42
42
|
entity_selector_diff: EntitySelectorDiff
|
|
43
43
|
errors: list[Exception]
|
|
44
|
+
number_of_transformed_entities: int = 0
|
|
44
45
|
misonfigured_entity_keys: dict[str, str] = field(default_factory=dict)
|
|
45
46
|
|
|
46
47
|
|
port_ocean/core/utils/utils.py
CHANGED
|
@@ -110,7 +110,8 @@ def get_port_diff(before: Iterable[Entity], after: Iterable[Entity]) -> EntityPo
|
|
|
110
110
|
|
|
111
111
|
|
|
112
112
|
def are_teams_different(
|
|
113
|
-
first_team: str | None | list[Any]
|
|
113
|
+
first_team: str | None | list[Any] | dict[str, Any],
|
|
114
|
+
second_team: str | None | list[Any] | dict[str, Any],
|
|
114
115
|
) -> bool:
|
|
115
116
|
if isinstance(first_team, list) and isinstance(second_team, list):
|
|
116
117
|
return sorted(first_team) != sorted(second_team)
|
|
@@ -122,6 +123,7 @@ def are_entities_fields_equal(
|
|
|
122
123
|
) -> bool:
|
|
123
124
|
"""
|
|
124
125
|
Compare two entity fields by serializing them to JSON and comparing their SHA-256 hashes.
|
|
126
|
+
Removes keys with None values before comparison if the corresponding key doesn't exist in the other dict.
|
|
125
127
|
|
|
126
128
|
Args:
|
|
127
129
|
first_entity_field: First entity field dictionary to compare
|
|
@@ -130,7 +132,13 @@ def are_entities_fields_equal(
|
|
|
130
132
|
Returns:
|
|
131
133
|
bool: True if the entity fields have identical content
|
|
132
134
|
"""
|
|
133
|
-
|
|
135
|
+
first_entity_field_copy = first_entity_field.copy()
|
|
136
|
+
|
|
137
|
+
for key in list(first_entity_field.keys()):
|
|
138
|
+
if first_entity_field[key] is None and key not in second_entity_field:
|
|
139
|
+
del first_entity_field_copy[key]
|
|
140
|
+
|
|
141
|
+
first_props = json.dumps(first_entity_field_copy, sort_keys=True)
|
|
134
142
|
second_props = json.dumps(second_entity_field, sort_keys=True)
|
|
135
143
|
first_hash = hashlib.sha256(first_props.encode()).hexdigest()
|
|
136
144
|
second_hash = hashlib.sha256(second_props.encode()).hexdigest()
|