port-ocean 0.21.4__py3-none-any.whl → 0.22.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of port-ocean might be problematic. Click here for more details.
- integrations/_infra/Makefile +2 -0
- port_ocean/cli/cookiecutter/cookiecutter.json +2 -2
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md +1 -1
- port_ocean/clients/port/mixins/entities.py +1 -2
- port_ocean/config/settings.py +22 -0
- port_ocean/context/event.py +1 -0
- port_ocean/context/ocean.py +5 -0
- port_ocean/context/resource.py +3 -4
- port_ocean/core/event_listener/base.py +6 -3
- port_ocean/core/handlers/entities_state_applier/port/applier.py +9 -1
- port_ocean/core/handlers/entity_processor/base.py +0 -2
- port_ocean/core/handlers/resync_state_updater/updater.py +9 -0
- port_ocean/core/handlers/webhook/processor_manager.py +5 -3
- port_ocean/core/handlers/webhook/webhook_event.py +0 -5
- port_ocean/core/integrations/mixins/sync_raw.py +61 -10
- port_ocean/core/ocean_types.py +1 -0
- port_ocean/helpers/metric/metric.py +238 -0
- port_ocean/helpers/metric/utils.py +30 -0
- port_ocean/helpers/retry.py +2 -1
- port_ocean/ocean.py +17 -4
- port_ocean/tests/clients/port/mixins/test_entities.py +11 -9
- port_ocean/tests/core/conftest.py +186 -0
- port_ocean/tests/core/handlers/entities_state_applier/test_applier.py +86 -6
- port_ocean/tests/core/handlers/mixins/test_sync_raw.py +5 -164
- port_ocean/tests/core/handlers/webhook/test_processor_manager.py +79 -44
- port_ocean/tests/test_metric.py +180 -0
- port_ocean/utils/async_http.py +4 -1
- {port_ocean-0.21.4.dist-info → port_ocean-0.22.0.dist-info}/METADATA +2 -1
- {port_ocean-0.21.4.dist-info → port_ocean-0.22.0.dist-info}/RECORD +32 -28
- {port_ocean-0.21.4.dist-info → port_ocean-0.22.0.dist-info}/LICENSE.md +0 -0
- {port_ocean-0.21.4.dist-info → port_ocean-0.22.0.dist-info}/WHEEL +0 -0
- {port_ocean-0.21.4.dist-info → port_ocean-0.22.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,9 +1,6 @@
|
|
|
1
|
-
from contextlib import asynccontextmanager
|
|
2
1
|
from graphlib import CycleError
|
|
3
|
-
from typing import Any
|
|
2
|
+
from typing import Any
|
|
4
3
|
|
|
5
|
-
from httpx import Response
|
|
6
|
-
from port_ocean.clients.port.client import PortClient
|
|
7
4
|
from port_ocean.core.utils.entity_topological_sorter import EntityTopologicalSorter
|
|
8
5
|
from port_ocean.exceptions.core import OceanAbortException
|
|
9
6
|
import pytest
|
|
@@ -11,12 +8,8 @@ from unittest.mock import MagicMock, AsyncMock, patch
|
|
|
11
8
|
from port_ocean.ocean import Ocean
|
|
12
9
|
from port_ocean.context.ocean import PortOceanContext
|
|
13
10
|
from port_ocean.core.handlers.port_app_config.models import (
|
|
14
|
-
EntityMapping,
|
|
15
|
-
MappingsConfig,
|
|
16
11
|
PortAppConfig,
|
|
17
|
-
PortResourceConfig,
|
|
18
12
|
ResourceConfig,
|
|
19
|
-
Selector,
|
|
20
13
|
)
|
|
21
14
|
from port_ocean.core.integrations.mixins import SyncRawMixin
|
|
22
15
|
from port_ocean.core.handlers.entities_state_applier.port.applier import (
|
|
@@ -26,148 +19,11 @@ from port_ocean.core.handlers.entity_processor.jq_entity_processor import (
|
|
|
26
19
|
JQEntityProcessor,
|
|
27
20
|
)
|
|
28
21
|
from port_ocean.core.models import Entity
|
|
29
|
-
from port_ocean.context.event import
|
|
22
|
+
from port_ocean.context.event import event_context, EventType
|
|
30
23
|
from port_ocean.clients.port.types import UserAgentType
|
|
31
|
-
from port_ocean.context.ocean import ocean
|
|
32
24
|
from dataclasses import dataclass
|
|
33
25
|
from typing import List, Optional
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
@pytest.fixture
|
|
37
|
-
def mock_port_client(mock_http_client: MagicMock) -> PortClient:
|
|
38
|
-
mock_port_client = PortClient(
|
|
39
|
-
MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
|
|
40
|
-
)
|
|
41
|
-
mock_port_client.auth = AsyncMock()
|
|
42
|
-
mock_port_client.auth.headers = AsyncMock(
|
|
43
|
-
return_value={
|
|
44
|
-
"Authorization": "test",
|
|
45
|
-
"User-Agent": "test",
|
|
46
|
-
}
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
mock_port_client.search_entities = AsyncMock(return_value=[]) # type: ignore
|
|
50
|
-
mock_port_client.client = mock_http_client
|
|
51
|
-
return mock_port_client
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
@pytest.fixture
|
|
55
|
-
def mock_http_client() -> MagicMock:
|
|
56
|
-
mock_http_client = MagicMock()
|
|
57
|
-
mock_upserted_entities = []
|
|
58
|
-
|
|
59
|
-
async def post(url: str, *args: Any, **kwargs: Any) -> Response:
|
|
60
|
-
entity = kwargs.get("json", {})
|
|
61
|
-
if entity.get("properties", {}).get("mock_is_to_fail", {}):
|
|
62
|
-
return Response(
|
|
63
|
-
404, headers=MagicMock(), json={"ok": False, "error": "not_found"}
|
|
64
|
-
)
|
|
65
|
-
|
|
66
|
-
mock_upserted_entities.append(
|
|
67
|
-
f"{entity.get('identifier')}-{entity.get('blueprint')}"
|
|
68
|
-
)
|
|
69
|
-
return Response(
|
|
70
|
-
200,
|
|
71
|
-
json={
|
|
72
|
-
"entity": {
|
|
73
|
-
"identifier": entity.get("identifier"),
|
|
74
|
-
"blueprint": entity.get("blueprint"),
|
|
75
|
-
}
|
|
76
|
-
},
|
|
77
|
-
)
|
|
78
|
-
|
|
79
|
-
mock_http_client.post = AsyncMock(side_effect=post)
|
|
80
|
-
return mock_http_client
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
@pytest.fixture
|
|
84
|
-
def mock_ocean(mock_port_client: PortClient) -> Ocean:
|
|
85
|
-
with patch("port_ocean.ocean.Ocean.__init__", return_value=None):
|
|
86
|
-
ocean_mock = Ocean(
|
|
87
|
-
MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
|
|
88
|
-
)
|
|
89
|
-
ocean_mock.config = MagicMock()
|
|
90
|
-
ocean_mock.config.port = MagicMock()
|
|
91
|
-
ocean_mock.config.port.port_app_config_cache_ttl = 60
|
|
92
|
-
ocean_mock.port_client = mock_port_client
|
|
93
|
-
|
|
94
|
-
return ocean_mock
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
@pytest.fixture
|
|
98
|
-
def mock_context(mock_ocean: Ocean) -> PortOceanContext:
|
|
99
|
-
context = PortOceanContext(mock_ocean)
|
|
100
|
-
ocean._app = context.app
|
|
101
|
-
return context
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
@pytest.fixture
|
|
105
|
-
def mock_port_app_config() -> PortAppConfig:
|
|
106
|
-
return PortAppConfig(
|
|
107
|
-
enable_merge_entity=True,
|
|
108
|
-
delete_dependent_entities=True,
|
|
109
|
-
create_missing_related_entities=False,
|
|
110
|
-
resources=[
|
|
111
|
-
ResourceConfig(
|
|
112
|
-
kind="project",
|
|
113
|
-
selector=Selector(query="true"),
|
|
114
|
-
port=PortResourceConfig(
|
|
115
|
-
entity=MappingsConfig(
|
|
116
|
-
mappings=EntityMapping(
|
|
117
|
-
identifier=".id | tostring",
|
|
118
|
-
title=".name",
|
|
119
|
-
blueprint='"service"',
|
|
120
|
-
properties={"url": ".web_url"},
|
|
121
|
-
relations={},
|
|
122
|
-
)
|
|
123
|
-
)
|
|
124
|
-
),
|
|
125
|
-
)
|
|
126
|
-
],
|
|
127
|
-
)
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
@pytest.fixture
|
|
131
|
-
def mock_port_app_config_handler(mock_port_app_config: PortAppConfig) -> MagicMock:
|
|
132
|
-
handler = MagicMock()
|
|
133
|
-
|
|
134
|
-
async def get_config(use_cache: bool = True) -> Any:
|
|
135
|
-
return mock_port_app_config
|
|
136
|
-
|
|
137
|
-
handler.get_port_app_config = get_config
|
|
138
|
-
return handler
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
@pytest.fixture
|
|
142
|
-
def mock_entity_processor(mock_context: PortOceanContext) -> JQEntityProcessor:
|
|
143
|
-
return JQEntityProcessor(mock_context)
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
@pytest.fixture
|
|
147
|
-
def mock_resource_config() -> ResourceConfig:
|
|
148
|
-
resource = ResourceConfig(
|
|
149
|
-
kind="service",
|
|
150
|
-
selector=Selector(query="true"),
|
|
151
|
-
port=PortResourceConfig(
|
|
152
|
-
entity=MappingsConfig(
|
|
153
|
-
mappings=EntityMapping(
|
|
154
|
-
identifier=".id",
|
|
155
|
-
title=".name",
|
|
156
|
-
blueprint='"service"',
|
|
157
|
-
properties={"url": ".web_url"},
|
|
158
|
-
relations={},
|
|
159
|
-
)
|
|
160
|
-
)
|
|
161
|
-
),
|
|
162
|
-
)
|
|
163
|
-
return resource
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
@pytest.fixture
|
|
167
|
-
def mock_entities_state_applier(
|
|
168
|
-
mock_context: PortOceanContext,
|
|
169
|
-
) -> HttpEntitiesStateApplier:
|
|
170
|
-
return HttpEntitiesStateApplier(mock_context)
|
|
26
|
+
from port_ocean.tests.core.conftest import create_entity, no_op_event_context
|
|
171
27
|
|
|
172
28
|
|
|
173
29
|
@pytest.fixture
|
|
@@ -189,27 +45,12 @@ def mock_sync_raw_mixin(
|
|
|
189
45
|
@pytest.fixture
|
|
190
46
|
def mock_sync_raw_mixin_with_jq_processor(
|
|
191
47
|
mock_sync_raw_mixin: SyncRawMixin,
|
|
48
|
+
mock_context: PortOceanContext,
|
|
192
49
|
) -> SyncRawMixin:
|
|
193
|
-
mock_sync_raw_mixin._entity_processor = JQEntityProcessor(mock_context)
|
|
50
|
+
mock_sync_raw_mixin._entity_processor = JQEntityProcessor(mock_context)
|
|
194
51
|
return mock_sync_raw_mixin
|
|
195
52
|
|
|
196
53
|
|
|
197
|
-
@asynccontextmanager
|
|
198
|
-
async def no_op_event_context(
|
|
199
|
-
existing_event: EventContext,
|
|
200
|
-
) -> AsyncGenerator[EventContext, None]:
|
|
201
|
-
yield existing_event
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
def create_entity(
|
|
205
|
-
id: str, blueprint: str, relation: dict[str, str], is_to_fail: bool
|
|
206
|
-
) -> Entity:
|
|
207
|
-
entity = Entity(identifier=id, blueprint=blueprint)
|
|
208
|
-
entity.relations = relation
|
|
209
|
-
entity.properties = {"mock_is_to_fail": is_to_fail}
|
|
210
|
-
return entity
|
|
211
|
-
|
|
212
|
-
|
|
213
54
|
@pytest.mark.asyncio
|
|
214
55
|
async def test_sync_raw_mixin_self_dependency(
|
|
215
56
|
mock_sync_raw_mixin: SyncRawMixin,
|
|
@@ -12,6 +12,7 @@ from port_ocean.core.handlers.webhook.webhook_event import (
|
|
|
12
12
|
EventPayload,
|
|
13
13
|
)
|
|
14
14
|
from fastapi import APIRouter
|
|
15
|
+
from port_ocean.core.integrations.mixins.handler import HandlerMixin
|
|
15
16
|
from port_ocean.utils.signal import SignalHandler
|
|
16
17
|
from typing import Dict, Any
|
|
17
18
|
import asyncio
|
|
@@ -19,7 +20,7 @@ from fastapi.testclient import TestClient
|
|
|
19
20
|
from fastapi import FastAPI
|
|
20
21
|
from port_ocean.context.ocean import PortOceanContext
|
|
21
22
|
from unittest.mock import AsyncMock
|
|
22
|
-
from port_ocean.context.event import event_context, EventType
|
|
23
|
+
from port_ocean.context.event import EventContext, event_context, EventType
|
|
23
24
|
from port_ocean.context.ocean import ocean
|
|
24
25
|
from unittest.mock import MagicMock, patch
|
|
25
26
|
from httpx import Response
|
|
@@ -571,6 +572,16 @@ async def test_integrationTest_postRequestSent_webhookEventRawResultProcessed_en
|
|
|
571
572
|
"sync_raw_results",
|
|
572
573
|
patched_export_single_resource,
|
|
573
574
|
)
|
|
575
|
+
monkeypatch.setattr(
|
|
576
|
+
HandlerMixin,
|
|
577
|
+
"port_app_config_handler",
|
|
578
|
+
AsyncMock(return_value=mock_port_app_config),
|
|
579
|
+
)
|
|
580
|
+
monkeypatch.setattr(
|
|
581
|
+
EventContext,
|
|
582
|
+
"port_app_config",
|
|
583
|
+
mock_port_app_config,
|
|
584
|
+
)
|
|
574
585
|
test_path = "/webhook-test"
|
|
575
586
|
mock_context.app.integration = BaseIntegration(ocean)
|
|
576
587
|
mock_context.app.webhook_manager = LiveEventsProcessorManager(
|
|
@@ -589,12 +600,7 @@ async def test_integrationTest_postRequestSent_webhookEventRawResultProcessed_en
|
|
|
589
600
|
|
|
590
601
|
test_payload = {"test": "data"}
|
|
591
602
|
|
|
592
|
-
async with event_context(EventType.HTTP_REQUEST, trigger_type="request")
|
|
593
|
-
mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config = AsyncMock(return_value=mock_port_app_config) # type: ignore
|
|
594
|
-
event.port_app_config = (
|
|
595
|
-
await mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config()
|
|
596
|
-
)
|
|
597
|
-
|
|
603
|
+
async with event_context(EventType.HTTP_REQUEST, trigger_type="request"):
|
|
598
604
|
response = client.post(
|
|
599
605
|
test_path, json=test_payload, headers={"Content-Type": "application/json"}
|
|
600
606
|
)
|
|
@@ -685,6 +691,16 @@ async def test_integrationTest_postRequestSent_reachedTimeout_entityNotUpserted(
|
|
|
685
691
|
"_process_single_event",
|
|
686
692
|
patched_process_single_event,
|
|
687
693
|
)
|
|
694
|
+
monkeypatch.setattr(
|
|
695
|
+
HandlerMixin,
|
|
696
|
+
"port_app_config_handler",
|
|
697
|
+
AsyncMock(return_value=mock_port_app_config),
|
|
698
|
+
)
|
|
699
|
+
monkeypatch.setattr(
|
|
700
|
+
EventContext,
|
|
701
|
+
"port_app_config",
|
|
702
|
+
mock_port_app_config,
|
|
703
|
+
)
|
|
688
704
|
test_path = "/webhook-test"
|
|
689
705
|
mock_context.app.integration = BaseIntegration(ocean)
|
|
690
706
|
mock_context.app.webhook_manager = LiveEventsProcessorManager(
|
|
@@ -703,12 +719,7 @@ async def test_integrationTest_postRequestSent_reachedTimeout_entityNotUpserted(
|
|
|
703
719
|
|
|
704
720
|
test_payload = {"test": "data"}
|
|
705
721
|
|
|
706
|
-
async with event_context(EventType.HTTP_REQUEST, trigger_type="request")
|
|
707
|
-
mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config = AsyncMock(return_value=mock_port_app_config) # type: ignore
|
|
708
|
-
event.port_app_config = (
|
|
709
|
-
await mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config()
|
|
710
|
-
)
|
|
711
|
-
|
|
722
|
+
async with event_context(EventType.HTTP_REQUEST, trigger_type="request"):
|
|
712
723
|
response = client.post(
|
|
713
724
|
test_path, json=test_payload, headers={"Content-Type": "application/json"}
|
|
714
725
|
)
|
|
@@ -802,6 +813,16 @@ async def test_integrationTest_postRequestSent_noMatchingHandlers_entityNotUpser
|
|
|
802
813
|
"_extract_matching_processors",
|
|
803
814
|
patched_extract_matching_processors,
|
|
804
815
|
)
|
|
816
|
+
monkeypatch.setattr(
|
|
817
|
+
HandlerMixin,
|
|
818
|
+
"port_app_config_handler",
|
|
819
|
+
AsyncMock(return_value=mock_port_app_config),
|
|
820
|
+
)
|
|
821
|
+
monkeypatch.setattr(
|
|
822
|
+
EventContext,
|
|
823
|
+
"port_app_config",
|
|
824
|
+
mock_port_app_config,
|
|
825
|
+
)
|
|
805
826
|
test_path = "/webhook-test"
|
|
806
827
|
mock_context.app.integration = BaseIntegration(ocean)
|
|
807
828
|
mock_context.app.webhook_manager = LiveEventsProcessorManager(
|
|
@@ -819,13 +840,7 @@ async def test_integrationTest_postRequestSent_noMatchingHandlers_entityNotUpser
|
|
|
819
840
|
client = TestClient(mock_context.app.fast_api_app)
|
|
820
841
|
|
|
821
842
|
test_payload = {"test": "data"}
|
|
822
|
-
|
|
823
|
-
async with event_context(EventType.HTTP_REQUEST, trigger_type="request") as event:
|
|
824
|
-
mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config = AsyncMock(return_value=mock_port_app_config) # type: ignore
|
|
825
|
-
event.port_app_config = (
|
|
826
|
-
await mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config()
|
|
827
|
-
)
|
|
828
|
-
|
|
843
|
+
async with event_context(EventType.HTTP_REQUEST, trigger_type="request"):
|
|
829
844
|
response = client.post(
|
|
830
845
|
test_path, json=test_payload, headers={"Content-Type": "application/json"}
|
|
831
846
|
)
|
|
@@ -982,6 +997,16 @@ async def test_integrationTest_postRequestSent_webhookEventRawResultProcessedFor
|
|
|
982
997
|
"sync_raw_results",
|
|
983
998
|
patched_export_single_resource,
|
|
984
999
|
)
|
|
1000
|
+
monkeypatch.setattr(
|
|
1001
|
+
HandlerMixin,
|
|
1002
|
+
"port_app_config_handler",
|
|
1003
|
+
AsyncMock(return_value=mock_port_app_config),
|
|
1004
|
+
)
|
|
1005
|
+
monkeypatch.setattr(
|
|
1006
|
+
EventContext,
|
|
1007
|
+
"port_app_config",
|
|
1008
|
+
mock_port_app_config,
|
|
1009
|
+
)
|
|
985
1010
|
test_path = "/webhook-test"
|
|
986
1011
|
mock_context.app.integration = BaseIntegration(ocean)
|
|
987
1012
|
mock_context.app.webhook_manager = LiveEventsProcessorManager(
|
|
@@ -1004,12 +1029,7 @@ async def test_integrationTest_postRequestSent_webhookEventRawResultProcessedFor
|
|
|
1004
1029
|
|
|
1005
1030
|
test_payload = {"test": "data"}
|
|
1006
1031
|
|
|
1007
|
-
async with event_context(EventType.HTTP_REQUEST, trigger_type="request")
|
|
1008
|
-
mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config = AsyncMock(return_value=mock_port_app_config) # type: ignore
|
|
1009
|
-
event.port_app_config = (
|
|
1010
|
-
await mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config()
|
|
1011
|
-
)
|
|
1012
|
-
|
|
1032
|
+
async with event_context(EventType.HTTP_REQUEST, trigger_type="request"):
|
|
1013
1033
|
response = client.post(
|
|
1014
1034
|
test_path, json=test_payload, headers={"Content-Type": "application/json"}
|
|
1015
1035
|
)
|
|
@@ -1112,6 +1132,16 @@ async def test_integrationTest_postRequestSent_webhookEventRawResultProcessedwit
|
|
|
1112
1132
|
"sync_raw_results",
|
|
1113
1133
|
patched_export_single_resource,
|
|
1114
1134
|
)
|
|
1135
|
+
monkeypatch.setattr(
|
|
1136
|
+
HandlerMixin,
|
|
1137
|
+
"port_app_config_handler",
|
|
1138
|
+
AsyncMock(return_value=mock_port_app_config),
|
|
1139
|
+
)
|
|
1140
|
+
monkeypatch.setattr(
|
|
1141
|
+
EventContext,
|
|
1142
|
+
"port_app_config",
|
|
1143
|
+
mock_port_app_config,
|
|
1144
|
+
)
|
|
1115
1145
|
test_path = "/webhook-test"
|
|
1116
1146
|
mock_context.app.integration = BaseIntegration(ocean)
|
|
1117
1147
|
mock_context.app.webhook_manager = LiveEventsProcessorManager(
|
|
@@ -1130,12 +1160,7 @@ async def test_integrationTest_postRequestSent_webhookEventRawResultProcessedwit
|
|
|
1130
1160
|
|
|
1131
1161
|
test_payload = {"test": "data"}
|
|
1132
1162
|
|
|
1133
|
-
async with event_context(EventType.HTTP_REQUEST, trigger_type="request")
|
|
1134
|
-
mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config = AsyncMock(return_value=mock_port_app_config) # type: ignore
|
|
1135
|
-
event.port_app_config = (
|
|
1136
|
-
await mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config()
|
|
1137
|
-
)
|
|
1138
|
-
|
|
1163
|
+
async with event_context(EventType.HTTP_REQUEST, trigger_type="request"):
|
|
1139
1164
|
response = client.post(
|
|
1140
1165
|
test_path, json=test_payload, headers={"Content-Type": "application/json"}
|
|
1141
1166
|
)
|
|
@@ -1242,6 +1267,16 @@ async def test_integrationTest_postRequestSent_webhookEventRawResultProcessedwit
|
|
|
1242
1267
|
"_process_webhook_request",
|
|
1243
1268
|
patched_process_webhook_request,
|
|
1244
1269
|
)
|
|
1270
|
+
monkeypatch.setattr(
|
|
1271
|
+
HandlerMixin,
|
|
1272
|
+
"port_app_config_handler",
|
|
1273
|
+
AsyncMock(return_value=mock_port_app_config),
|
|
1274
|
+
)
|
|
1275
|
+
monkeypatch.setattr(
|
|
1276
|
+
EventContext,
|
|
1277
|
+
"port_app_config",
|
|
1278
|
+
mock_port_app_config,
|
|
1279
|
+
)
|
|
1245
1280
|
test_path = "/webhook-test"
|
|
1246
1281
|
mock_context.app.integration = BaseIntegration(ocean)
|
|
1247
1282
|
mock_context.app.webhook_manager = LiveEventsProcessorManager(
|
|
@@ -1260,12 +1295,7 @@ async def test_integrationTest_postRequestSent_webhookEventRawResultProcessedwit
|
|
|
1260
1295
|
|
|
1261
1296
|
test_payload = {"test": "data"}
|
|
1262
1297
|
|
|
1263
|
-
async with event_context(EventType.HTTP_REQUEST, trigger_type="request")
|
|
1264
|
-
mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config = AsyncMock(return_value=mock_port_app_config) # type: ignore
|
|
1265
|
-
event.port_app_config = (
|
|
1266
|
-
await mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config()
|
|
1267
|
-
)
|
|
1268
|
-
|
|
1298
|
+
async with event_context(EventType.HTTP_REQUEST, trigger_type="request"):
|
|
1269
1299
|
response = client.post(
|
|
1270
1300
|
test_path, json=test_payload, headers={"Content-Type": "application/json"}
|
|
1271
1301
|
)
|
|
@@ -1381,6 +1411,16 @@ async def test_integrationTest_postRequestSent_oneProcessorThrowsException_onlyS
|
|
|
1381
1411
|
"sync_raw_results",
|
|
1382
1412
|
patched_export_single_resource,
|
|
1383
1413
|
)
|
|
1414
|
+
monkeypatch.setattr(
|
|
1415
|
+
HandlerMixin,
|
|
1416
|
+
"port_app_config_handler",
|
|
1417
|
+
AsyncMock(return_value=mock_port_app_config),
|
|
1418
|
+
)
|
|
1419
|
+
monkeypatch.setattr(
|
|
1420
|
+
EventContext,
|
|
1421
|
+
"port_app_config",
|
|
1422
|
+
mock_port_app_config,
|
|
1423
|
+
)
|
|
1384
1424
|
test_path = "/webhook-test"
|
|
1385
1425
|
mock_context.app.integration = BaseIntegration(ocean)
|
|
1386
1426
|
mock_context.app.webhook_manager = LiveEventsProcessorManager(
|
|
@@ -1401,12 +1441,7 @@ async def test_integrationTest_postRequestSent_oneProcessorThrowsException_onlyS
|
|
|
1401
1441
|
|
|
1402
1442
|
test_payload = {"test": "data"}
|
|
1403
1443
|
|
|
1404
|
-
async with event_context(EventType.HTTP_REQUEST, trigger_type="request")
|
|
1405
|
-
mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config = AsyncMock(return_value=mock_port_app_config) # type: ignore
|
|
1406
|
-
event.port_app_config = (
|
|
1407
|
-
await mock_context.app.webhook_manager.port_app_config_handler.get_port_app_config()
|
|
1408
|
-
)
|
|
1409
|
-
|
|
1444
|
+
async with event_context(EventType.HTTP_REQUEST, trigger_type="request"):
|
|
1410
1445
|
response = client.post(
|
|
1411
1446
|
test_path, json=test_payload, headers={"Content-Type": "application/json"}
|
|
1412
1447
|
)
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@pytest.mark.metric
|
|
6
|
+
@pytest.mark.skip(reason="Skipping metric test until we have a way to test the metrics")
|
|
7
|
+
def test_metrics() -> None:
|
|
8
|
+
"""
|
|
9
|
+
Test that the metrics logged in /tmp/ocean/metric.log match expected values.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
log_path = "/tmp/ocean/metric.log"
|
|
13
|
+
delay = 2
|
|
14
|
+
batch_size = 400
|
|
15
|
+
total_objects = 2000
|
|
16
|
+
magic_string = "prometheus metrics |"
|
|
17
|
+
|
|
18
|
+
# Read the file
|
|
19
|
+
with open(log_path, "r") as file:
|
|
20
|
+
content = file.read()
|
|
21
|
+
|
|
22
|
+
# Ensure the magic string is present in the content
|
|
23
|
+
assert magic_string in content, f"'{magic_string}' not found in {log_path}"
|
|
24
|
+
|
|
25
|
+
# Isolate and parse the JSON object after the magic string
|
|
26
|
+
start_idx = content.rfind(magic_string)
|
|
27
|
+
content_after_magic = content[start_idx + len(magic_string) :]
|
|
28
|
+
obj = ast.literal_eval(content_after_magic)
|
|
29
|
+
|
|
30
|
+
# ----------------------------------------------------------------------------
|
|
31
|
+
# 1. Validate Extract Duration (using original delay/batch_size logic)
|
|
32
|
+
# ----------------------------------------------------------------------------
|
|
33
|
+
num_batches = total_objects / batch_size # e.g., 2000 / 400 = 5
|
|
34
|
+
expected_min_extract_duration = num_batches * delay # e.g., 5 * 2 = 10
|
|
35
|
+
|
|
36
|
+
# Check "fake-person-1" extract duration is > expected_min_extract_duration
|
|
37
|
+
actual_extract_duration = obj.get("duration_seconds__fake-person-1__extract", 0)
|
|
38
|
+
assert round(actual_extract_duration) > round(expected_min_extract_duration), (
|
|
39
|
+
f"Extract duration {actual_extract_duration} not greater than "
|
|
40
|
+
f"{expected_min_extract_duration}"
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# ----------------------------------------------------------------------------
|
|
44
|
+
# 2. Check Durations for Both "fake-person-1" and "fake-department-0"
|
|
45
|
+
# ----------------------------------------------------------------------------
|
|
46
|
+
# -- fake-person-1
|
|
47
|
+
transform_duration_p1 = obj.get("duration_seconds__fake-person-1__transform", 0)
|
|
48
|
+
load_duration_p1 = obj.get("duration_seconds__fake-person-1__load", 0)
|
|
49
|
+
assert (
|
|
50
|
+
transform_duration_p1 > 0
|
|
51
|
+
), f"Expected transform duration > 0, got {transform_duration_p1}"
|
|
52
|
+
assert load_duration_p1 > 0, f"Expected load duration > 0, got {load_duration_p1}"
|
|
53
|
+
|
|
54
|
+
# -- fake-department-0
|
|
55
|
+
extract_duration_dept0 = obj.get("duration_seconds__fake-department-0__extract", 0)
|
|
56
|
+
transform_duration_dept0 = obj.get(
|
|
57
|
+
"duration_seconds__fake-department-0__transform", 0
|
|
58
|
+
)
|
|
59
|
+
load_duration_dept0 = obj.get("duration_seconds__fake-department-0__load", 0)
|
|
60
|
+
|
|
61
|
+
assert (
|
|
62
|
+
extract_duration_dept0 > 0
|
|
63
|
+
), f"Expected department extract duration > 0, got {extract_duration_dept0}"
|
|
64
|
+
assert (
|
|
65
|
+
transform_duration_dept0 > 0
|
|
66
|
+
), f"Expected department transform duration > 0, got {transform_duration_dept0}"
|
|
67
|
+
assert (
|
|
68
|
+
load_duration_dept0 > 0
|
|
69
|
+
), f"Expected department load duration > 0, got {load_duration_dept0}"
|
|
70
|
+
|
|
71
|
+
# Optionally, check the "init__top_sort" duration too, if it's relevant:
|
|
72
|
+
init_top_sort = obj.get("duration_seconds__init__top_sort", 0)
|
|
73
|
+
assert init_top_sort >= 0, f"Expected init__top_sort >= 0, got {init_top_sort}"
|
|
74
|
+
|
|
75
|
+
# ----------------------------------------------------------------------------
|
|
76
|
+
# 3. Check Object Counts
|
|
77
|
+
# ----------------------------------------------------------------------------
|
|
78
|
+
# -- fake-person-1
|
|
79
|
+
person_extract_count = obj.get("object_count__fake-person-1__extract", 0)
|
|
80
|
+
person_load_count = obj.get("object_count__fake-person-1__load", 0)
|
|
81
|
+
assert person_extract_count == 2000.0, (
|
|
82
|
+
f"Expected object_count__fake-person-1__extract=2000.0, "
|
|
83
|
+
f"got {person_extract_count}"
|
|
84
|
+
)
|
|
85
|
+
assert person_load_count == 4000.0, (
|
|
86
|
+
f"Expected object_count__fake-person-1__load=4000.0, "
|
|
87
|
+
f"got {person_load_count}"
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# -- fake-department-0
|
|
91
|
+
dept_extract_count = obj.get("object_count__fake-department-0__extract", 0)
|
|
92
|
+
dept_load_count = obj.get("object_count__fake-department-0__load", 0)
|
|
93
|
+
assert dept_extract_count == 5.0, (
|
|
94
|
+
f"Expected object_count__fake-department-0__extract=5.0, "
|
|
95
|
+
f"got {dept_extract_count}"
|
|
96
|
+
)
|
|
97
|
+
assert dept_load_count == 10.0, (
|
|
98
|
+
f"Expected object_count__fake-department-0__load=10.0, "
|
|
99
|
+
f"got {dept_load_count}"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# ----------------------------------------------------------------------------
|
|
103
|
+
# 4. Check Input/Upserted Counts
|
|
104
|
+
# ----------------------------------------------------------------------------
|
|
105
|
+
# -- fake-person-1
|
|
106
|
+
input_count_p1 = obj.get("input_count__fake-person-1__load", 0)
|
|
107
|
+
upserted_count_p1 = obj.get("upserted_count__fake-person-1__load", 0)
|
|
108
|
+
assert (
|
|
109
|
+
input_count_p1 == 2000.0
|
|
110
|
+
), f"Expected input_count__fake-person-1__load=2000.0, got {input_count_p1}"
|
|
111
|
+
assert (
|
|
112
|
+
upserted_count_p1 == 2000.0
|
|
113
|
+
), f"Expected upserted_count__fake-person-1__load=2000.0, got {upserted_count_p1}"
|
|
114
|
+
|
|
115
|
+
# -- fake-department-0
|
|
116
|
+
input_count_dept0 = obj.get("input_count__fake-department-0__load", 0)
|
|
117
|
+
upserted_count_dept0 = obj.get("upserted_count__fake-department-0__load", 0)
|
|
118
|
+
assert (
|
|
119
|
+
input_count_dept0 == 5.0
|
|
120
|
+
), f"Expected input_count__fake-department-0__load=5.0, got {input_count_dept0}"
|
|
121
|
+
assert (
|
|
122
|
+
upserted_count_dept0 == 5.0
|
|
123
|
+
), f"Expected upserted_count__fake-department-0__load=5.0, got {upserted_count_dept0}"
|
|
124
|
+
|
|
125
|
+
# ----------------------------------------------------------------------------
|
|
126
|
+
# 5. Check Error and Failed Counts
|
|
127
|
+
# ----------------------------------------------------------------------------
|
|
128
|
+
# -- fake-person-1
|
|
129
|
+
error_count_p1 = obj.get("error_count__fake-person-1__load", 0)
|
|
130
|
+
failed_count_p1 = obj.get("failed_count__fake-person-1__load", 0)
|
|
131
|
+
assert (
|
|
132
|
+
error_count_p1 == 0.0
|
|
133
|
+
), f"Expected error_count__fake-person-1__load=0.0, got {error_count_p1}"
|
|
134
|
+
assert (
|
|
135
|
+
failed_count_p1 == 0.0
|
|
136
|
+
), f"Expected failed_count__fake-person-1__load=0.0, got {failed_count_p1}"
|
|
137
|
+
|
|
138
|
+
# -- fake-department-0
|
|
139
|
+
error_count_dept0 = obj.get("error_count__fake-department-0__load", 0)
|
|
140
|
+
failed_count_dept0 = obj.get("failed_count__fake-department-0__load", 0)
|
|
141
|
+
assert (
|
|
142
|
+
error_count_dept0 == 0.0
|
|
143
|
+
), f"Expected error_count__fake-department-0__load=0.0, got {error_count_dept0}"
|
|
144
|
+
assert (
|
|
145
|
+
failed_count_dept0 == 0.0
|
|
146
|
+
), f"Expected failed_count__fake-department-0__load=0.0, got {failed_count_dept0}"
|
|
147
|
+
|
|
148
|
+
# ----------------------------------------------------------------------------
|
|
149
|
+
# 6. Check HTTP Request Counts (200s)
|
|
150
|
+
# ----------------------------------------------------------------------------
|
|
151
|
+
# Example: we confirm certain request counters match the sample data provided:
|
|
152
|
+
assert (
|
|
153
|
+
obj.get(
|
|
154
|
+
"http_requests_count__http://host.docker.internal:5555/v1/auth/access_token__init__load__200",
|
|
155
|
+
0,
|
|
156
|
+
)
|
|
157
|
+
== 1.0
|
|
158
|
+
), "Expected 1.0 for auth access_token 200 requests"
|
|
159
|
+
assert (
|
|
160
|
+
obj.get(
|
|
161
|
+
"http_requests_count__http://host.docker.internal:5555/v1/integration/smoke-test-integration__init__load__200",
|
|
162
|
+
0,
|
|
163
|
+
)
|
|
164
|
+
== 5.0
|
|
165
|
+
), "Expected 5.0 for integration/smoke-test-integration 200 requests"
|
|
166
|
+
assert (
|
|
167
|
+
obj.get(
|
|
168
|
+
"http_requests_count__http://localhost:8000/integration/department/hr/employees?limit=-1&entity_kb_size=1&latency=2000__fake-person-1__extract__200",
|
|
169
|
+
0,
|
|
170
|
+
)
|
|
171
|
+
== 1.0
|
|
172
|
+
), "Expected 1.0 for hr/employees?limit=-1 extract 200 requests"
|
|
173
|
+
expected_requests = {
|
|
174
|
+
"http_requests_count__http://localhost:8000/integration/department/marketing/employees?limit=-1&entity_kb_size=1&latency=2000__fake-person-1__extract__200": 1.0,
|
|
175
|
+
"http_requests_count__http://localhost:8000/integration/department/finance/employees?limit=-1&entity_kb_size=1&latency=2000__fake-person-1__extract__200": 1.0,
|
|
176
|
+
}
|
|
177
|
+
for key, expected_val in expected_requests.items():
|
|
178
|
+
assert (
|
|
179
|
+
obj.get(key, 0) == expected_val
|
|
180
|
+
), f"Expected {expected_val} for '{key}', got {obj.get(key)}"
|
port_ocean/utils/async_http.py
CHANGED
|
@@ -11,7 +11,10 @@ _http_client: LocalStack[httpx.AsyncClient] = LocalStack()
|
|
|
11
11
|
def _get_http_client_context() -> httpx.AsyncClient:
|
|
12
12
|
client = _http_client.top
|
|
13
13
|
if client is None:
|
|
14
|
-
client = OceanAsyncClient(
|
|
14
|
+
client = OceanAsyncClient(
|
|
15
|
+
RetryTransport,
|
|
16
|
+
timeout=ocean.config.client_timeout,
|
|
17
|
+
)
|
|
15
18
|
_http_client.push(client)
|
|
16
19
|
|
|
17
20
|
return client
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: port-ocean
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.22.0
|
|
4
4
|
Summary: Port Ocean is a CLI tool for managing your Port projects.
|
|
5
5
|
Home-page: https://app.getport.io
|
|
6
6
|
Keywords: ocean,port-ocean,port
|
|
@@ -31,6 +31,7 @@ Requires-Dist: httpx (>=0.24.1,<0.28.0)
|
|
|
31
31
|
Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
|
|
32
32
|
Requires-Dist: jq (>=1.8.0,<2.0.0)
|
|
33
33
|
Requires-Dist: loguru (>=0.7.0,<0.8.0)
|
|
34
|
+
Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
|
|
34
35
|
Requires-Dist: pydantic[dotenv] (>=1.10.8,<2.0.0)
|
|
35
36
|
Requires-Dist: pydispatcher (>=2.0.7,<3.0.0)
|
|
36
37
|
Requires-Dist: pyhumps (>=3.8.0,<4.0.0)
|