port-ocean 0.21.5__py3-none-any.whl → 0.22.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of port-ocean might be problematic. Click here for more details.
- integrations/_infra/Makefile +2 -0
- port_ocean/cli/cookiecutter/cookiecutter.json +2 -2
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +1 -1
- port_ocean/clients/port/mixins/entities.py +11 -12
- port_ocean/config/settings.py +22 -0
- port_ocean/context/event.py +1 -0
- port_ocean/context/ocean.py +5 -0
- port_ocean/context/resource.py +3 -4
- port_ocean/core/defaults/common.py +1 -0
- port_ocean/core/defaults/initialize.py +1 -1
- port_ocean/core/event_listener/base.py +6 -3
- port_ocean/core/handlers/entities_state_applier/port/applier.py +23 -21
- port_ocean/core/handlers/entity_processor/base.py +0 -2
- port_ocean/core/handlers/port_app_config/models.py +1 -1
- port_ocean/core/handlers/resync_state_updater/updater.py +9 -0
- port_ocean/core/integrations/mixins/sync_raw.py +61 -10
- port_ocean/core/models.py +6 -2
- port_ocean/core/ocean_types.py +1 -0
- port_ocean/core/utils/utils.py +10 -2
- port_ocean/helpers/metric/metric.py +238 -0
- port_ocean/helpers/metric/utils.py +30 -0
- port_ocean/helpers/retry.py +2 -1
- port_ocean/ocean.py +17 -4
- port_ocean/tests/clients/port/mixins/test_entities.py +12 -9
- port_ocean/tests/core/conftest.py +187 -0
- port_ocean/tests/core/handlers/entities_state_applier/test_applier.py +154 -6
- port_ocean/tests/core/handlers/mixins/test_sync_raw.py +29 -164
- port_ocean/tests/core/utils/test_resolve_entities_diff.py +52 -0
- port_ocean/tests/test_metric.py +180 -0
- port_ocean/utils/async_http.py +4 -1
- {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/METADATA +2 -1
- {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/RECORD +35 -31
- {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/LICENSE.md +0 -0
- {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/WHEEL +0 -0
- {port_ocean-0.21.5.dist-info → port_ocean-0.22.1.dist-info}/entry_points.txt +0 -0
|
@@ -6,6 +6,11 @@ from port_ocean.core.handlers.entities_state_applier.port.applier import (
|
|
|
6
6
|
from port_ocean.core.models import Entity
|
|
7
7
|
from port_ocean.core.ocean_types import EntityDiff
|
|
8
8
|
from port_ocean.clients.port.types import UserAgentType
|
|
9
|
+
from port_ocean.ocean import Ocean
|
|
10
|
+
from port_ocean.context.ocean import PortOceanContext
|
|
11
|
+
from port_ocean.tests.core.conftest import create_entity
|
|
12
|
+
from port_ocean.core.handlers.port_app_config.models import PortAppConfig
|
|
13
|
+
from port_ocean.context.event import event_context, EventType
|
|
9
14
|
|
|
10
15
|
|
|
11
16
|
@pytest.mark.asyncio
|
|
@@ -23,8 +28,8 @@ async def test_delete_diff_no_deleted_entities() -> None:
|
|
|
23
28
|
|
|
24
29
|
|
|
25
30
|
@pytest.mark.asyncio
|
|
26
|
-
async def test_delete_diff_below_threshold() -> None:
|
|
27
|
-
applier = HttpEntitiesStateApplier(
|
|
31
|
+
async def test_delete_diff_below_threshold(mock_context: PortOceanContext) -> None:
|
|
32
|
+
applier = HttpEntitiesStateApplier(mock_context)
|
|
28
33
|
entities = EntityDiff(
|
|
29
34
|
before=[
|
|
30
35
|
Entity(identifier="1", blueprint="test"),
|
|
@@ -48,8 +53,10 @@ async def test_delete_diff_below_threshold() -> None:
|
|
|
48
53
|
|
|
49
54
|
|
|
50
55
|
@pytest.mark.asyncio
|
|
51
|
-
async def test_delete_diff_above_default_threshold(
|
|
52
|
-
|
|
56
|
+
async def test_delete_diff_above_default_threshold(
|
|
57
|
+
mock_context: PortOceanContext,
|
|
58
|
+
) -> None:
|
|
59
|
+
applier = HttpEntitiesStateApplier(mock_context)
|
|
53
60
|
entities = EntityDiff(
|
|
54
61
|
before=[
|
|
55
62
|
Entity(identifier="1", blueprint="test"),
|
|
@@ -68,8 +75,10 @@ async def test_delete_diff_above_default_threshold() -> None:
|
|
|
68
75
|
|
|
69
76
|
|
|
70
77
|
@pytest.mark.asyncio
|
|
71
|
-
async def test_delete_diff_custom_threshold_above_threshold_not_deleted(
|
|
72
|
-
|
|
78
|
+
async def test_delete_diff_custom_threshold_above_threshold_not_deleted(
|
|
79
|
+
mock_context: PortOceanContext,
|
|
80
|
+
) -> None:
|
|
81
|
+
applier = HttpEntitiesStateApplier(mock_context)
|
|
73
82
|
entities = EntityDiff(
|
|
74
83
|
before=[
|
|
75
84
|
Entity(identifier="1", blueprint="test"),
|
|
@@ -84,3 +93,142 @@ async def test_delete_diff_custom_threshold_above_threshold_not_deleted() -> Non
|
|
|
84
93
|
)
|
|
85
94
|
|
|
86
95
|
mock_safe_delete.assert_not_called()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@pytest.mark.asyncio
|
|
99
|
+
async def test_applier_with_mock_context(
|
|
100
|
+
mock_ocean: Ocean,
|
|
101
|
+
mock_context: PortOceanContext,
|
|
102
|
+
mock_port_app_config: PortAppConfig,
|
|
103
|
+
) -> None:
|
|
104
|
+
# Create an applier using the mock_context fixture
|
|
105
|
+
applier = HttpEntitiesStateApplier(mock_context)
|
|
106
|
+
|
|
107
|
+
# Create test entities
|
|
108
|
+
entity = Entity(identifier="test_entity", blueprint="test_blueprint")
|
|
109
|
+
|
|
110
|
+
async with event_context(EventType.RESYNC, trigger_type="machine") as event:
|
|
111
|
+
event.port_app_config = mock_port_app_config
|
|
112
|
+
|
|
113
|
+
# Test the upsert method with mocked client
|
|
114
|
+
with patch.object(mock_ocean.port_client.client, "post") as mock_post:
|
|
115
|
+
mock_post.return_value = Mock(
|
|
116
|
+
status_code=200,
|
|
117
|
+
json=lambda: {
|
|
118
|
+
"entity": {
|
|
119
|
+
"identifier": "test_entity",
|
|
120
|
+
"blueprint": "test_blueprint",
|
|
121
|
+
}
|
|
122
|
+
},
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
result = await applier.upsert([entity], UserAgentType.exporter)
|
|
126
|
+
|
|
127
|
+
# Assert that the post method was called
|
|
128
|
+
mock_post.assert_called_once()
|
|
129
|
+
assert len(result) == 1
|
|
130
|
+
assert result[0].identifier == "test_entity"
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@pytest.mark.asyncio
|
|
134
|
+
async def test_applier_one_not_upserted(
|
|
135
|
+
mock_ocean: Ocean,
|
|
136
|
+
mock_context: PortOceanContext,
|
|
137
|
+
mock_port_app_config: PortAppConfig,
|
|
138
|
+
) -> None:
|
|
139
|
+
# Create an applier using the mock_context fixture
|
|
140
|
+
applier = HttpEntitiesStateApplier(mock_context)
|
|
141
|
+
|
|
142
|
+
# Create test entities
|
|
143
|
+
entity = Entity(identifier="test_entity", blueprint="test_blueprint")
|
|
144
|
+
|
|
145
|
+
async with event_context(EventType.RESYNC, trigger_type="machine") as event:
|
|
146
|
+
# Mock the register_entity method
|
|
147
|
+
event.entity_topological_sorter.register_entity = Mock() # type: ignore
|
|
148
|
+
event.port_app_config = mock_port_app_config
|
|
149
|
+
|
|
150
|
+
# Test the upsert method with mocked client
|
|
151
|
+
with patch.object(mock_ocean.port_client.client, "post") as mock_post:
|
|
152
|
+
mock_post.return_value = Mock(
|
|
153
|
+
status_code=404,
|
|
154
|
+
json=lambda: {"ok": False, "error": "not_found"},
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
result = await applier.upsert([entity], UserAgentType.exporter)
|
|
158
|
+
|
|
159
|
+
# Assert that the post method was called
|
|
160
|
+
mock_post.assert_called_once()
|
|
161
|
+
assert len(result) == 0
|
|
162
|
+
event.entity_topological_sorter.register_entity.assert_called_once_with(
|
|
163
|
+
entity
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@pytest.mark.asyncio
|
|
168
|
+
async def test_applier_error_upserting(
|
|
169
|
+
mock_ocean: Ocean,
|
|
170
|
+
mock_context: PortOceanContext,
|
|
171
|
+
mock_port_app_config: PortAppConfig,
|
|
172
|
+
) -> None:
|
|
173
|
+
# Create an applier using the mock_context fixture
|
|
174
|
+
applier = HttpEntitiesStateApplier(mock_context)
|
|
175
|
+
|
|
176
|
+
# Create test entities
|
|
177
|
+
entity = Entity(identifier="test_entity", blueprint="test_blueprint")
|
|
178
|
+
|
|
179
|
+
async with event_context(EventType.RESYNC, trigger_type="machine") as event:
|
|
180
|
+
# Mock the register_entity method
|
|
181
|
+
event.entity_topological_sorter.register_entity = Mock() # type: ignore
|
|
182
|
+
event.port_app_config = mock_port_app_config
|
|
183
|
+
|
|
184
|
+
# Test the upsert method with mocked client
|
|
185
|
+
with patch.object(mock_ocean.port_client.client, "post") as mock_post:
|
|
186
|
+
mock_post.return_value = Mock(
|
|
187
|
+
status_code=404,
|
|
188
|
+
json=lambda: {"ok": False, "error": "not_found"},
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
result = await applier.upsert([entity], UserAgentType.exporter)
|
|
192
|
+
|
|
193
|
+
# Assert that the post method was called
|
|
194
|
+
mock_post.assert_called_once()
|
|
195
|
+
assert len(result) == 0
|
|
196
|
+
event.entity_topological_sorter.register_entity.assert_called_once_with(
|
|
197
|
+
entity
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@pytest.mark.asyncio
|
|
202
|
+
async def test_using_create_entity_helper(
|
|
203
|
+
mock_ocean: Ocean,
|
|
204
|
+
mock_context: PortOceanContext,
|
|
205
|
+
mock_port_app_config: PortAppConfig,
|
|
206
|
+
) -> None:
|
|
207
|
+
# Create the applier with the mock context
|
|
208
|
+
applier = HttpEntitiesStateApplier(mock_context)
|
|
209
|
+
|
|
210
|
+
# Create test entities using the helper function
|
|
211
|
+
entity1 = create_entity("entity1", "service", {"related_to": "entity2"}, False)
|
|
212
|
+
|
|
213
|
+
# Test that entities were created correctly
|
|
214
|
+
assert entity1.identifier == "entity1"
|
|
215
|
+
assert entity1.blueprint == "service"
|
|
216
|
+
assert entity1.relations == {"related_to": "entity2"}
|
|
217
|
+
assert entity1.properties == {"mock_is_to_fail": False}
|
|
218
|
+
|
|
219
|
+
# Test the applier with these entities
|
|
220
|
+
async with event_context(EventType.RESYNC, trigger_type="machine") as event:
|
|
221
|
+
event.port_app_config = mock_port_app_config
|
|
222
|
+
|
|
223
|
+
with patch.object(mock_ocean.port_client.client, "post") as mock_post:
|
|
224
|
+
mock_post.return_value = Mock(
|
|
225
|
+
status_code=200,
|
|
226
|
+
json=lambda: {
|
|
227
|
+
"entity": {"identifier": "entity1", "blueprint": "service"}
|
|
228
|
+
},
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
result = await applier.upsert([entity1], UserAgentType.exporter)
|
|
232
|
+
|
|
233
|
+
mock_post.assert_called_once()
|
|
234
|
+
assert len(result) == 1
|
|
@@ -1,9 +1,6 @@
|
|
|
1
|
-
from contextlib import asynccontextmanager
|
|
2
1
|
from graphlib import CycleError
|
|
3
|
-
from typing import Any
|
|
2
|
+
from typing import Any
|
|
4
3
|
|
|
5
|
-
from httpx import Response
|
|
6
|
-
from port_ocean.clients.port.client import PortClient
|
|
7
4
|
from port_ocean.core.utils.entity_topological_sorter import EntityTopologicalSorter
|
|
8
5
|
from port_ocean.exceptions.core import OceanAbortException
|
|
9
6
|
import pytest
|
|
@@ -11,12 +8,8 @@ from unittest.mock import MagicMock, AsyncMock, patch
|
|
|
11
8
|
from port_ocean.ocean import Ocean
|
|
12
9
|
from port_ocean.context.ocean import PortOceanContext
|
|
13
10
|
from port_ocean.core.handlers.port_app_config.models import (
|
|
14
|
-
EntityMapping,
|
|
15
|
-
MappingsConfig,
|
|
16
11
|
PortAppConfig,
|
|
17
|
-
PortResourceConfig,
|
|
18
12
|
ResourceConfig,
|
|
19
|
-
Selector,
|
|
20
13
|
)
|
|
21
14
|
from port_ocean.core.integrations.mixins import SyncRawMixin
|
|
22
15
|
from port_ocean.core.handlers.entities_state_applier.port.applier import (
|
|
@@ -26,148 +19,11 @@ from port_ocean.core.handlers.entity_processor.jq_entity_processor import (
|
|
|
26
19
|
JQEntityProcessor,
|
|
27
20
|
)
|
|
28
21
|
from port_ocean.core.models import Entity
|
|
29
|
-
from port_ocean.context.event import
|
|
22
|
+
from port_ocean.context.event import event_context, EventType
|
|
30
23
|
from port_ocean.clients.port.types import UserAgentType
|
|
31
|
-
from port_ocean.context.ocean import ocean
|
|
32
24
|
from dataclasses import dataclass
|
|
33
25
|
from typing import List, Optional
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
@pytest.fixture
|
|
37
|
-
def mock_port_client(mock_http_client: MagicMock) -> PortClient:
|
|
38
|
-
mock_port_client = PortClient(
|
|
39
|
-
MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
|
|
40
|
-
)
|
|
41
|
-
mock_port_client.auth = AsyncMock()
|
|
42
|
-
mock_port_client.auth.headers = AsyncMock(
|
|
43
|
-
return_value={
|
|
44
|
-
"Authorization": "test",
|
|
45
|
-
"User-Agent": "test",
|
|
46
|
-
}
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
mock_port_client.search_entities = AsyncMock(return_value=[]) # type: ignore
|
|
50
|
-
mock_port_client.client = mock_http_client
|
|
51
|
-
return mock_port_client
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
@pytest.fixture
|
|
55
|
-
def mock_http_client() -> MagicMock:
|
|
56
|
-
mock_http_client = MagicMock()
|
|
57
|
-
mock_upserted_entities = []
|
|
58
|
-
|
|
59
|
-
async def post(url: str, *args: Any, **kwargs: Any) -> Response:
|
|
60
|
-
entity = kwargs.get("json", {})
|
|
61
|
-
if entity.get("properties", {}).get("mock_is_to_fail", {}):
|
|
62
|
-
return Response(
|
|
63
|
-
404, headers=MagicMock(), json={"ok": False, "error": "not_found"}
|
|
64
|
-
)
|
|
65
|
-
|
|
66
|
-
mock_upserted_entities.append(
|
|
67
|
-
f"{entity.get('identifier')}-{entity.get('blueprint')}"
|
|
68
|
-
)
|
|
69
|
-
return Response(
|
|
70
|
-
200,
|
|
71
|
-
json={
|
|
72
|
-
"entity": {
|
|
73
|
-
"identifier": entity.get("identifier"),
|
|
74
|
-
"blueprint": entity.get("blueprint"),
|
|
75
|
-
}
|
|
76
|
-
},
|
|
77
|
-
)
|
|
78
|
-
|
|
79
|
-
mock_http_client.post = AsyncMock(side_effect=post)
|
|
80
|
-
return mock_http_client
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
@pytest.fixture
|
|
84
|
-
def mock_ocean(mock_port_client: PortClient) -> Ocean:
|
|
85
|
-
with patch("port_ocean.ocean.Ocean.__init__", return_value=None):
|
|
86
|
-
ocean_mock = Ocean(
|
|
87
|
-
MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
|
|
88
|
-
)
|
|
89
|
-
ocean_mock.config = MagicMock()
|
|
90
|
-
ocean_mock.config.port = MagicMock()
|
|
91
|
-
ocean_mock.config.port.port_app_config_cache_ttl = 60
|
|
92
|
-
ocean_mock.port_client = mock_port_client
|
|
93
|
-
|
|
94
|
-
return ocean_mock
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
@pytest.fixture
|
|
98
|
-
def mock_context(mock_ocean: Ocean) -> PortOceanContext:
|
|
99
|
-
context = PortOceanContext(mock_ocean)
|
|
100
|
-
ocean._app = context.app
|
|
101
|
-
return context
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
@pytest.fixture
|
|
105
|
-
def mock_port_app_config() -> PortAppConfig:
|
|
106
|
-
return PortAppConfig(
|
|
107
|
-
enable_merge_entity=True,
|
|
108
|
-
delete_dependent_entities=True,
|
|
109
|
-
create_missing_related_entities=False,
|
|
110
|
-
resources=[
|
|
111
|
-
ResourceConfig(
|
|
112
|
-
kind="project",
|
|
113
|
-
selector=Selector(query="true"),
|
|
114
|
-
port=PortResourceConfig(
|
|
115
|
-
entity=MappingsConfig(
|
|
116
|
-
mappings=EntityMapping(
|
|
117
|
-
identifier=".id | tostring",
|
|
118
|
-
title=".name",
|
|
119
|
-
blueprint='"service"',
|
|
120
|
-
properties={"url": ".web_url"},
|
|
121
|
-
relations={},
|
|
122
|
-
)
|
|
123
|
-
)
|
|
124
|
-
),
|
|
125
|
-
)
|
|
126
|
-
],
|
|
127
|
-
)
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
@pytest.fixture
|
|
131
|
-
def mock_port_app_config_handler(mock_port_app_config: PortAppConfig) -> MagicMock:
|
|
132
|
-
handler = MagicMock()
|
|
133
|
-
|
|
134
|
-
async def get_config(use_cache: bool = True) -> Any:
|
|
135
|
-
return mock_port_app_config
|
|
136
|
-
|
|
137
|
-
handler.get_port_app_config = get_config
|
|
138
|
-
return handler
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
@pytest.fixture
|
|
142
|
-
def mock_entity_processor(mock_context: PortOceanContext) -> JQEntityProcessor:
|
|
143
|
-
return JQEntityProcessor(mock_context)
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
@pytest.fixture
|
|
147
|
-
def mock_resource_config() -> ResourceConfig:
|
|
148
|
-
resource = ResourceConfig(
|
|
149
|
-
kind="service",
|
|
150
|
-
selector=Selector(query="true"),
|
|
151
|
-
port=PortResourceConfig(
|
|
152
|
-
entity=MappingsConfig(
|
|
153
|
-
mappings=EntityMapping(
|
|
154
|
-
identifier=".id",
|
|
155
|
-
title=".name",
|
|
156
|
-
blueprint='"service"',
|
|
157
|
-
properties={"url": ".web_url"},
|
|
158
|
-
relations={},
|
|
159
|
-
)
|
|
160
|
-
)
|
|
161
|
-
),
|
|
162
|
-
)
|
|
163
|
-
return resource
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
@pytest.fixture
|
|
167
|
-
def mock_entities_state_applier(
|
|
168
|
-
mock_context: PortOceanContext,
|
|
169
|
-
) -> HttpEntitiesStateApplier:
|
|
170
|
-
return HttpEntitiesStateApplier(mock_context)
|
|
26
|
+
from port_ocean.tests.core.conftest import create_entity, no_op_event_context
|
|
171
27
|
|
|
172
28
|
|
|
173
29
|
@pytest.fixture
|
|
@@ -189,27 +45,12 @@ def mock_sync_raw_mixin(
|
|
|
189
45
|
@pytest.fixture
|
|
190
46
|
def mock_sync_raw_mixin_with_jq_processor(
|
|
191
47
|
mock_sync_raw_mixin: SyncRawMixin,
|
|
48
|
+
mock_context: PortOceanContext,
|
|
192
49
|
) -> SyncRawMixin:
|
|
193
|
-
mock_sync_raw_mixin._entity_processor = JQEntityProcessor(mock_context)
|
|
50
|
+
mock_sync_raw_mixin._entity_processor = JQEntityProcessor(mock_context)
|
|
194
51
|
return mock_sync_raw_mixin
|
|
195
52
|
|
|
196
53
|
|
|
197
|
-
@asynccontextmanager
|
|
198
|
-
async def no_op_event_context(
|
|
199
|
-
existing_event: EventContext,
|
|
200
|
-
) -> AsyncGenerator[EventContext, None]:
|
|
201
|
-
yield existing_event
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
def create_entity(
|
|
205
|
-
id: str, blueprint: str, relation: dict[str, str], is_to_fail: bool
|
|
206
|
-
) -> Entity:
|
|
207
|
-
entity = Entity(identifier=id, blueprint=blueprint)
|
|
208
|
-
entity.relations = relation
|
|
209
|
-
entity.properties = {"mock_is_to_fail": is_to_fail}
|
|
210
|
-
return entity
|
|
211
|
-
|
|
212
|
-
|
|
213
54
|
@pytest.mark.asyncio
|
|
214
55
|
async def test_sync_raw_mixin_self_dependency(
|
|
215
56
|
mock_sync_raw_mixin: SyncRawMixin,
|
|
@@ -576,6 +417,30 @@ async def test_map_entities_compared_with_port_no_port_entities_all_entities_are
|
|
|
576
417
|
assert "entity_2" in [e.identifier for e in changed_entities]
|
|
577
418
|
|
|
578
419
|
|
|
420
|
+
@pytest.mark.asyncio
|
|
421
|
+
async def test_map_entities_compared_with_port_returns_original_entities_when_using_team_search_query(
|
|
422
|
+
mock_sync_raw_mixin: SyncRawMixin,
|
|
423
|
+
mock_resource_config: ResourceConfig,
|
|
424
|
+
) -> None:
|
|
425
|
+
|
|
426
|
+
team_search_query = {
|
|
427
|
+
"combinator": "and",
|
|
428
|
+
"rules": [{"property": "$team", "operator": "=", "value": "my-team"}],
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
entities = [
|
|
432
|
+
create_entity("entity_1", "service", {}, False, team=team_search_query),
|
|
433
|
+
create_entity("entity_2", "service", {}, False, team=team_search_query),
|
|
434
|
+
]
|
|
435
|
+
|
|
436
|
+
changed_entities = await mock_sync_raw_mixin._map_entities_compared_with_port(
|
|
437
|
+
entities, mock_resource_config, UserAgentType.exporter
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
assert len(changed_entities) == 2
|
|
441
|
+
assert changed_entities == entities
|
|
442
|
+
|
|
443
|
+
|
|
579
444
|
@pytest.mark.asyncio
|
|
580
445
|
async def test_map_entities_compared_with_port_with_existing_entities_only_changed_third_party_entities_are_mapped(
|
|
581
446
|
mock_sync_raw_mixin: SyncRawMixin,
|
|
@@ -246,6 +246,58 @@ def test_are_entities_fields_equal_different_nested_relations_should_be_false()
|
|
|
246
246
|
)
|
|
247
247
|
|
|
248
248
|
|
|
249
|
+
def test_are_entities_fields_equal_null_properties_should_be_true() -> None:
|
|
250
|
+
assert (
|
|
251
|
+
are_entities_fields_equal(
|
|
252
|
+
{
|
|
253
|
+
"team": None,
|
|
254
|
+
"members": ["user1", "user2"],
|
|
255
|
+
"metadata": {"role": "admin"},
|
|
256
|
+
},
|
|
257
|
+
{"members": ["user1", "user2"], "metadata": {"role": "admin"}},
|
|
258
|
+
)
|
|
259
|
+
is True
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def test_are_entities_fields_equal_null_properties_and_real_data_in_other_entity_should_be_false() -> (
|
|
264
|
+
None
|
|
265
|
+
):
|
|
266
|
+
assert (
|
|
267
|
+
are_entities_fields_equal(
|
|
268
|
+
{
|
|
269
|
+
"team": None,
|
|
270
|
+
"members": ["user1", "user2"],
|
|
271
|
+
"metadata": {"role": "admin"},
|
|
272
|
+
},
|
|
273
|
+
{
|
|
274
|
+
"team": "team_id1",
|
|
275
|
+
"members": ["user1", "user2"],
|
|
276
|
+
"metadata": {"role": "admin"},
|
|
277
|
+
},
|
|
278
|
+
)
|
|
279
|
+
is False
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def test_are_entities_fields_equal_null_properties_in_both_should_be_true() -> None:
|
|
284
|
+
assert (
|
|
285
|
+
are_entities_fields_equal(
|
|
286
|
+
{
|
|
287
|
+
"team": None,
|
|
288
|
+
"members": ["user1", "user2"],
|
|
289
|
+
"metadata": {"role": "admin"},
|
|
290
|
+
},
|
|
291
|
+
{
|
|
292
|
+
"team": None,
|
|
293
|
+
"members": ["user1", "user2"],
|
|
294
|
+
"metadata": {"role": "admin"},
|
|
295
|
+
},
|
|
296
|
+
)
|
|
297
|
+
is True
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
|
|
249
301
|
def test_are_entities_different_identical_entities_should_be_false() -> None:
|
|
250
302
|
entity1 = create_test_entity(
|
|
251
303
|
"",
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@pytest.mark.metric
|
|
6
|
+
@pytest.mark.skip(reason="Skipping metric test until we have a way to test the metrics")
|
|
7
|
+
def test_metrics() -> None:
|
|
8
|
+
"""
|
|
9
|
+
Test that the metrics logged in /tmp/ocean/metric.log match expected values.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
log_path = "/tmp/ocean/metric.log"
|
|
13
|
+
delay = 2
|
|
14
|
+
batch_size = 400
|
|
15
|
+
total_objects = 2000
|
|
16
|
+
magic_string = "prometheus metrics |"
|
|
17
|
+
|
|
18
|
+
# Read the file
|
|
19
|
+
with open(log_path, "r") as file:
|
|
20
|
+
content = file.read()
|
|
21
|
+
|
|
22
|
+
# Ensure the magic string is present in the content
|
|
23
|
+
assert magic_string in content, f"'{magic_string}' not found in {log_path}"
|
|
24
|
+
|
|
25
|
+
# Isolate and parse the JSON object after the magic string
|
|
26
|
+
start_idx = content.rfind(magic_string)
|
|
27
|
+
content_after_magic = content[start_idx + len(magic_string) :]
|
|
28
|
+
obj = ast.literal_eval(content_after_magic)
|
|
29
|
+
|
|
30
|
+
# ----------------------------------------------------------------------------
|
|
31
|
+
# 1. Validate Extract Duration (using original delay/batch_size logic)
|
|
32
|
+
# ----------------------------------------------------------------------------
|
|
33
|
+
num_batches = total_objects / batch_size # e.g., 2000 / 400 = 5
|
|
34
|
+
expected_min_extract_duration = num_batches * delay # e.g., 5 * 2 = 10
|
|
35
|
+
|
|
36
|
+
# Check "fake-person-1" extract duration is > expected_min_extract_duration
|
|
37
|
+
actual_extract_duration = obj.get("duration_seconds__fake-person-1__extract", 0)
|
|
38
|
+
assert round(actual_extract_duration) > round(expected_min_extract_duration), (
|
|
39
|
+
f"Extract duration {actual_extract_duration} not greater than "
|
|
40
|
+
f"{expected_min_extract_duration}"
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# ----------------------------------------------------------------------------
|
|
44
|
+
# 2. Check Durations for Both "fake-person-1" and "fake-department-0"
|
|
45
|
+
# ----------------------------------------------------------------------------
|
|
46
|
+
# -- fake-person-1
|
|
47
|
+
transform_duration_p1 = obj.get("duration_seconds__fake-person-1__transform", 0)
|
|
48
|
+
load_duration_p1 = obj.get("duration_seconds__fake-person-1__load", 0)
|
|
49
|
+
assert (
|
|
50
|
+
transform_duration_p1 > 0
|
|
51
|
+
), f"Expected transform duration > 0, got {transform_duration_p1}"
|
|
52
|
+
assert load_duration_p1 > 0, f"Expected load duration > 0, got {load_duration_p1}"
|
|
53
|
+
|
|
54
|
+
# -- fake-department-0
|
|
55
|
+
extract_duration_dept0 = obj.get("duration_seconds__fake-department-0__extract", 0)
|
|
56
|
+
transform_duration_dept0 = obj.get(
|
|
57
|
+
"duration_seconds__fake-department-0__transform", 0
|
|
58
|
+
)
|
|
59
|
+
load_duration_dept0 = obj.get("duration_seconds__fake-department-0__load", 0)
|
|
60
|
+
|
|
61
|
+
assert (
|
|
62
|
+
extract_duration_dept0 > 0
|
|
63
|
+
), f"Expected department extract duration > 0, got {extract_duration_dept0}"
|
|
64
|
+
assert (
|
|
65
|
+
transform_duration_dept0 > 0
|
|
66
|
+
), f"Expected department transform duration > 0, got {transform_duration_dept0}"
|
|
67
|
+
assert (
|
|
68
|
+
load_duration_dept0 > 0
|
|
69
|
+
), f"Expected department load duration > 0, got {load_duration_dept0}"
|
|
70
|
+
|
|
71
|
+
# Optionally, check the "init__top_sort" duration too, if it's relevant:
|
|
72
|
+
init_top_sort = obj.get("duration_seconds__init__top_sort", 0)
|
|
73
|
+
assert init_top_sort >= 0, f"Expected init__top_sort >= 0, got {init_top_sort}"
|
|
74
|
+
|
|
75
|
+
# ----------------------------------------------------------------------------
|
|
76
|
+
# 3. Check Object Counts
|
|
77
|
+
# ----------------------------------------------------------------------------
|
|
78
|
+
# -- fake-person-1
|
|
79
|
+
person_extract_count = obj.get("object_count__fake-person-1__extract", 0)
|
|
80
|
+
person_load_count = obj.get("object_count__fake-person-1__load", 0)
|
|
81
|
+
assert person_extract_count == 2000.0, (
|
|
82
|
+
f"Expected object_count__fake-person-1__extract=2000.0, "
|
|
83
|
+
f"got {person_extract_count}"
|
|
84
|
+
)
|
|
85
|
+
assert person_load_count == 4000.0, (
|
|
86
|
+
f"Expected object_count__fake-person-1__load=4000.0, "
|
|
87
|
+
f"got {person_load_count}"
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# -- fake-department-0
|
|
91
|
+
dept_extract_count = obj.get("object_count__fake-department-0__extract", 0)
|
|
92
|
+
dept_load_count = obj.get("object_count__fake-department-0__load", 0)
|
|
93
|
+
assert dept_extract_count == 5.0, (
|
|
94
|
+
f"Expected object_count__fake-department-0__extract=5.0, "
|
|
95
|
+
f"got {dept_extract_count}"
|
|
96
|
+
)
|
|
97
|
+
assert dept_load_count == 10.0, (
|
|
98
|
+
f"Expected object_count__fake-department-0__load=10.0, "
|
|
99
|
+
f"got {dept_load_count}"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# ----------------------------------------------------------------------------
|
|
103
|
+
# 4. Check Input/Upserted Counts
|
|
104
|
+
# ----------------------------------------------------------------------------
|
|
105
|
+
# -- fake-person-1
|
|
106
|
+
input_count_p1 = obj.get("input_count__fake-person-1__load", 0)
|
|
107
|
+
upserted_count_p1 = obj.get("upserted_count__fake-person-1__load", 0)
|
|
108
|
+
assert (
|
|
109
|
+
input_count_p1 == 2000.0
|
|
110
|
+
), f"Expected input_count__fake-person-1__load=2000.0, got {input_count_p1}"
|
|
111
|
+
assert (
|
|
112
|
+
upserted_count_p1 == 2000.0
|
|
113
|
+
), f"Expected upserted_count__fake-person-1__load=2000.0, got {upserted_count_p1}"
|
|
114
|
+
|
|
115
|
+
# -- fake-department-0
|
|
116
|
+
input_count_dept0 = obj.get("input_count__fake-department-0__load", 0)
|
|
117
|
+
upserted_count_dept0 = obj.get("upserted_count__fake-department-0__load", 0)
|
|
118
|
+
assert (
|
|
119
|
+
input_count_dept0 == 5.0
|
|
120
|
+
), f"Expected input_count__fake-department-0__load=5.0, got {input_count_dept0}"
|
|
121
|
+
assert (
|
|
122
|
+
upserted_count_dept0 == 5.0
|
|
123
|
+
), f"Expected upserted_count__fake-department-0__load=5.0, got {upserted_count_dept0}"
|
|
124
|
+
|
|
125
|
+
# ----------------------------------------------------------------------------
|
|
126
|
+
# 5. Check Error and Failed Counts
|
|
127
|
+
# ----------------------------------------------------------------------------
|
|
128
|
+
# -- fake-person-1
|
|
129
|
+
error_count_p1 = obj.get("error_count__fake-person-1__load", 0)
|
|
130
|
+
failed_count_p1 = obj.get("failed_count__fake-person-1__load", 0)
|
|
131
|
+
assert (
|
|
132
|
+
error_count_p1 == 0.0
|
|
133
|
+
), f"Expected error_count__fake-person-1__load=0.0, got {error_count_p1}"
|
|
134
|
+
assert (
|
|
135
|
+
failed_count_p1 == 0.0
|
|
136
|
+
), f"Expected failed_count__fake-person-1__load=0.0, got {failed_count_p1}"
|
|
137
|
+
|
|
138
|
+
# -- fake-department-0
|
|
139
|
+
error_count_dept0 = obj.get("error_count__fake-department-0__load", 0)
|
|
140
|
+
failed_count_dept0 = obj.get("failed_count__fake-department-0__load", 0)
|
|
141
|
+
assert (
|
|
142
|
+
error_count_dept0 == 0.0
|
|
143
|
+
), f"Expected error_count__fake-department-0__load=0.0, got {error_count_dept0}"
|
|
144
|
+
assert (
|
|
145
|
+
failed_count_dept0 == 0.0
|
|
146
|
+
), f"Expected failed_count__fake-department-0__load=0.0, got {failed_count_dept0}"
|
|
147
|
+
|
|
148
|
+
# ----------------------------------------------------------------------------
|
|
149
|
+
# 6. Check HTTP Request Counts (200s)
|
|
150
|
+
# ----------------------------------------------------------------------------
|
|
151
|
+
# Example: we confirm certain request counters match the sample data provided:
|
|
152
|
+
assert (
|
|
153
|
+
obj.get(
|
|
154
|
+
"http_requests_count__http://host.docker.internal:5555/v1/auth/access_token__init__load__200",
|
|
155
|
+
0,
|
|
156
|
+
)
|
|
157
|
+
== 1.0
|
|
158
|
+
), "Expected 1.0 for auth access_token 200 requests"
|
|
159
|
+
assert (
|
|
160
|
+
obj.get(
|
|
161
|
+
"http_requests_count__http://host.docker.internal:5555/v1/integration/smoke-test-integration__init__load__200",
|
|
162
|
+
0,
|
|
163
|
+
)
|
|
164
|
+
== 5.0
|
|
165
|
+
), "Expected 5.0 for integration/smoke-test-integration 200 requests"
|
|
166
|
+
assert (
|
|
167
|
+
obj.get(
|
|
168
|
+
"http_requests_count__http://localhost:8000/integration/department/hr/employees?limit=-1&entity_kb_size=1&latency=2000__fake-person-1__extract__200",
|
|
169
|
+
0,
|
|
170
|
+
)
|
|
171
|
+
== 1.0
|
|
172
|
+
), "Expected 1.0 for hr/employees?limit=-1 extract 200 requests"
|
|
173
|
+
expected_requests = {
|
|
174
|
+
"http_requests_count__http://localhost:8000/integration/department/marketing/employees?limit=-1&entity_kb_size=1&latency=2000__fake-person-1__extract__200": 1.0,
|
|
175
|
+
"http_requests_count__http://localhost:8000/integration/department/finance/employees?limit=-1&entity_kb_size=1&latency=2000__fake-person-1__extract__200": 1.0,
|
|
176
|
+
}
|
|
177
|
+
for key, expected_val in expected_requests.items():
|
|
178
|
+
assert (
|
|
179
|
+
obj.get(key, 0) == expected_val
|
|
180
|
+
), f"Expected {expected_val} for '{key}', got {obj.get(key)}"
|
port_ocean/utils/async_http.py
CHANGED
|
@@ -11,7 +11,10 @@ _http_client: LocalStack[httpx.AsyncClient] = LocalStack()
|
|
|
11
11
|
def _get_http_client_context() -> httpx.AsyncClient:
|
|
12
12
|
client = _http_client.top
|
|
13
13
|
if client is None:
|
|
14
|
-
client = OceanAsyncClient(
|
|
14
|
+
client = OceanAsyncClient(
|
|
15
|
+
RetryTransport,
|
|
16
|
+
timeout=ocean.config.client_timeout,
|
|
17
|
+
)
|
|
15
18
|
_http_client.push(client)
|
|
16
19
|
|
|
17
20
|
return client
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: port-ocean
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.22.1
|
|
4
4
|
Summary: Port Ocean is a CLI tool for managing your Port projects.
|
|
5
5
|
Home-page: https://app.getport.io
|
|
6
6
|
Keywords: ocean,port-ocean,port
|
|
@@ -31,6 +31,7 @@ Requires-Dist: httpx (>=0.24.1,<0.28.0)
|
|
|
31
31
|
Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
|
|
32
32
|
Requires-Dist: jq (>=1.8.0,<2.0.0)
|
|
33
33
|
Requires-Dist: loguru (>=0.7.0,<0.8.0)
|
|
34
|
+
Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
|
|
34
35
|
Requires-Dist: pydantic[dotenv] (>=1.10.8,<2.0.0)
|
|
35
36
|
Requires-Dist: pydispatcher (>=2.0.7,<3.0.0)
|
|
36
37
|
Requires-Dist: pyhumps (>=3.8.0,<4.0.0)
|