port-ocean 0.5.6__py3-none-any.whl → 0.17.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of port-ocean might be problematic. Click here for more details.
- integrations/_infra/Dockerfile.Deb +56 -0
- integrations/_infra/Dockerfile.alpine +108 -0
- integrations/_infra/Dockerfile.base.builder +26 -0
- integrations/_infra/Dockerfile.base.runner +13 -0
- integrations/_infra/Dockerfile.dockerignore +94 -0
- {port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}} → integrations/_infra}/Makefile +21 -8
- integrations/_infra/grpcio.sh +18 -0
- integrations/_infra/init.sh +5 -0
- port_ocean/bootstrap.py +1 -1
- port_ocean/cli/commands/defaults/clean.py +3 -1
- port_ocean/cli/commands/new.py +42 -7
- port_ocean/cli/commands/sail.py +7 -1
- port_ocean/cli/cookiecutter/cookiecutter.json +3 -0
- port_ocean/cli/cookiecutter/hooks/post_gen_project.py +20 -3
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.env.example +6 -0
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/blueprints.json +41 -0
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/port-app-config.yml +16 -0
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml +6 -7
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md +1 -1
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CONTRIBUTING.md +7 -0
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore +1 -0
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/main.py +16 -1
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml +21 -10
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/test_sample.py +2 -0
- port_ocean/clients/port/authentication.py +16 -4
- port_ocean/clients/port/client.py +17 -0
- port_ocean/clients/port/mixins/blueprints.py +7 -8
- port_ocean/clients/port/mixins/entities.py +108 -53
- port_ocean/clients/port/mixins/integrations.py +23 -34
- port_ocean/clients/port/retry_transport.py +0 -5
- port_ocean/clients/port/utils.py +9 -3
- port_ocean/config/base.py +16 -16
- port_ocean/config/settings.py +79 -11
- port_ocean/context/event.py +18 -5
- port_ocean/context/ocean.py +14 -3
- port_ocean/core/defaults/clean.py +10 -3
- port_ocean/core/defaults/common.py +25 -9
- port_ocean/core/defaults/initialize.py +111 -100
- port_ocean/core/event_listener/__init__.py +8 -0
- port_ocean/core/event_listener/base.py +49 -10
- port_ocean/core/event_listener/factory.py +9 -1
- port_ocean/core/event_listener/http.py +11 -3
- port_ocean/core/event_listener/kafka.py +24 -5
- port_ocean/core/event_listener/once.py +96 -4
- port_ocean/core/event_listener/polling.py +16 -14
- port_ocean/core/event_listener/webhooks_only.py +41 -0
- port_ocean/core/handlers/__init__.py +1 -2
- port_ocean/core/handlers/entities_state_applier/base.py +4 -1
- port_ocean/core/handlers/entities_state_applier/port/applier.py +29 -87
- port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py +5 -2
- port_ocean/core/handlers/entity_processor/base.py +26 -22
- port_ocean/core/handlers/entity_processor/jq_entity_processor.py +253 -45
- port_ocean/core/handlers/port_app_config/base.py +55 -15
- port_ocean/core/handlers/port_app_config/models.py +24 -5
- port_ocean/core/handlers/resync_state_updater/__init__.py +5 -0
- port_ocean/core/handlers/resync_state_updater/updater.py +84 -0
- port_ocean/core/integrations/base.py +5 -7
- port_ocean/core/integrations/mixins/events.py +3 -1
- port_ocean/core/integrations/mixins/sync.py +4 -2
- port_ocean/core/integrations/mixins/sync_raw.py +209 -74
- port_ocean/core/integrations/mixins/utils.py +1 -1
- port_ocean/core/models.py +44 -0
- port_ocean/core/ocean_types.py +29 -11
- port_ocean/core/utils/entity_topological_sorter.py +90 -0
- port_ocean/core/utils/utils.py +109 -0
- port_ocean/debug_cli.py +5 -0
- port_ocean/exceptions/core.py +4 -0
- port_ocean/exceptions/port_defaults.py +0 -2
- port_ocean/helpers/retry.py +85 -24
- port_ocean/log/handlers.py +23 -2
- port_ocean/log/logger_setup.py +8 -1
- port_ocean/log/sensetive.py +25 -10
- port_ocean/middlewares.py +10 -2
- port_ocean/ocean.py +57 -24
- port_ocean/run.py +10 -5
- port_ocean/tests/__init__.py +0 -0
- port_ocean/tests/clients/port/mixins/test_entities.py +53 -0
- port_ocean/tests/conftest.py +4 -0
- port_ocean/tests/core/defaults/test_common.py +166 -0
- port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py +350 -0
- port_ocean/tests/core/handlers/mixins/test_sync_raw.py +552 -0
- port_ocean/tests/core/test_utils.py +73 -0
- port_ocean/tests/core/utils/test_entity_topological_sorter.py +99 -0
- port_ocean/tests/helpers/__init__.py +0 -0
- port_ocean/tests/helpers/fake_port_api.py +191 -0
- port_ocean/tests/helpers/fixtures.py +46 -0
- port_ocean/tests/helpers/integration.py +31 -0
- port_ocean/tests/helpers/ocean_app.py +66 -0
- port_ocean/tests/helpers/port_client.py +21 -0
- port_ocean/tests/helpers/smoke_test.py +82 -0
- port_ocean/tests/log/test_handlers.py +71 -0
- port_ocean/tests/test_smoke.py +74 -0
- port_ocean/tests/utils/test_async_iterators.py +45 -0
- port_ocean/tests/utils/test_cache.py +189 -0
- port_ocean/utils/async_iterators.py +109 -0
- port_ocean/utils/cache.py +37 -1
- port_ocean/utils/misc.py +22 -4
- port_ocean/utils/queue_utils.py +88 -0
- port_ocean/utils/signal.py +1 -4
- port_ocean/utils/time.py +54 -0
- {port_ocean-0.5.6.dist-info → port_ocean-0.17.8.dist-info}/METADATA +27 -19
- port_ocean-0.17.8.dist-info/RECORD +164 -0
- {port_ocean-0.5.6.dist-info → port_ocean-0.17.8.dist-info}/WHEEL +1 -1
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.dockerignore +0 -94
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Dockerfile +0 -15
- port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/config.yaml +0 -17
- port_ocean/core/handlers/entities_state_applier/port/validate_entity_relations.py +0 -40
- port_ocean/core/utils.py +0 -65
- port_ocean-0.5.6.dist-info/RECORD +0 -129
- {port_ocean-0.5.6.dist-info → port_ocean-0.17.8.dist-info}/LICENSE.md +0 -0
- {port_ocean-0.5.6.dist-info → port_ocean-0.17.8.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from typing import Iterable, Any, TypeVar, Callable, Awaitable
|
|
3
|
+
|
|
4
|
+
from loguru import logger
|
|
5
|
+
from pydantic import parse_obj_as, ValidationError
|
|
6
|
+
|
|
7
|
+
from port_ocean.clients.port.client import PortClient
|
|
8
|
+
from port_ocean.core.models import Entity, Runtime
|
|
9
|
+
from port_ocean.core.models import EntityPortDiff
|
|
10
|
+
from port_ocean.core.ocean_types import RAW_RESULT
|
|
11
|
+
from port_ocean.exceptions.core import (
|
|
12
|
+
RawObjectValidationException,
|
|
13
|
+
IntegrationRuntimeException,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
T = TypeVar("T", bound=tuple[list[Any], ...])
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def zip_and_sum(collection: Iterable[T]) -> T:
|
|
20
|
+
return tuple(sum(items, []) for items in zip(*collection)) # type: ignore
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def validate_result(result: Any) -> RAW_RESULT:
|
|
24
|
+
try:
|
|
25
|
+
return parse_obj_as(list[dict[str, Any]], result)
|
|
26
|
+
except ValidationError as e:
|
|
27
|
+
raise RawObjectValidationException(f"Expected list[dict[str, Any]], Error: {e}")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def is_same_entity(first_entity: Entity, second_entity: Entity) -> bool:
|
|
31
|
+
return (
|
|
32
|
+
first_entity.identifier == second_entity.identifier
|
|
33
|
+
and first_entity.blueprint == second_entity.blueprint
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
async def validate_integration_runtime(
|
|
38
|
+
port_client: PortClient,
|
|
39
|
+
requested_runtime: Runtime,
|
|
40
|
+
) -> None:
|
|
41
|
+
logger.debug("Validating integration runtime")
|
|
42
|
+
current_integration = await port_client.get_current_integration(
|
|
43
|
+
should_raise=False, should_log=False
|
|
44
|
+
)
|
|
45
|
+
current_installation_type = current_integration.get("installationType", "OnPrem")
|
|
46
|
+
if current_integration and not requested_runtime.is_installation_type_compatible(
|
|
47
|
+
current_installation_type
|
|
48
|
+
):
|
|
49
|
+
raise IntegrationRuntimeException(
|
|
50
|
+
f"Invalid Runtime! Requested to run existing {current_installation_type} integration in {requested_runtime} runtime."
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
Q = TypeVar("Q")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
async def gather_and_split_errors_from_results(
|
|
58
|
+
task: Iterable[Awaitable[Q]],
|
|
59
|
+
result_threshold_validation: Callable[[Q | Exception], bool] | None = None,
|
|
60
|
+
) -> tuple[list[Q], list[Exception]]:
|
|
61
|
+
valid_items: list[Q] = []
|
|
62
|
+
errors: list[Exception] = []
|
|
63
|
+
results = await asyncio.gather(*task, return_exceptions=True)
|
|
64
|
+
for item in results:
|
|
65
|
+
# return_exceptions will also catch Python BaseException which also includes KeyboardInterrupt, SystemExit, GeneratorExit
|
|
66
|
+
# https://docs.python.org/3/library/asyncio-task.html#asyncio.gather
|
|
67
|
+
# These exceptions should be raised and not caught for the application to exit properly.
|
|
68
|
+
# https://stackoverflow.com/a/17802352
|
|
69
|
+
if isinstance(item, BaseException) and not isinstance(item, Exception):
|
|
70
|
+
raise item
|
|
71
|
+
elif isinstance(item, Exception):
|
|
72
|
+
errors.append(item)
|
|
73
|
+
elif not result_threshold_validation or result_threshold_validation(item):
|
|
74
|
+
valid_items.append(item)
|
|
75
|
+
|
|
76
|
+
return valid_items, errors
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def get_port_diff(
|
|
80
|
+
before: Iterable[Entity],
|
|
81
|
+
after: Iterable[Entity],
|
|
82
|
+
) -> EntityPortDiff:
|
|
83
|
+
before_dict = {}
|
|
84
|
+
after_dict = {}
|
|
85
|
+
created = []
|
|
86
|
+
modified = []
|
|
87
|
+
deleted = []
|
|
88
|
+
|
|
89
|
+
# Create dictionaries for before and after lists
|
|
90
|
+
for entity in before:
|
|
91
|
+
key = (entity.identifier, entity.blueprint)
|
|
92
|
+
before_dict[key] = entity
|
|
93
|
+
|
|
94
|
+
for entity in after:
|
|
95
|
+
key = (entity.identifier, entity.blueprint)
|
|
96
|
+
after_dict[key] = entity
|
|
97
|
+
|
|
98
|
+
# Find created, modified, and deleted objects
|
|
99
|
+
for key, obj in after_dict.items():
|
|
100
|
+
if key not in before_dict:
|
|
101
|
+
created.append(obj)
|
|
102
|
+
else:
|
|
103
|
+
modified.append(obj)
|
|
104
|
+
|
|
105
|
+
for key, obj in before_dict.items():
|
|
106
|
+
if key not in after_dict:
|
|
107
|
+
deleted.append(obj)
|
|
108
|
+
|
|
109
|
+
return EntityPortDiff(created=created, modified=modified, deleted=deleted)
|
port_ocean/debug_cli.py
ADDED
port_ocean/exceptions/core.py
CHANGED
|
@@ -6,10 +6,8 @@ class AbortDefaultCreationError(BaseOceanException):
|
|
|
6
6
|
self,
|
|
7
7
|
blueprints_to_rollback: list[str],
|
|
8
8
|
errors: list[Exception],
|
|
9
|
-
pages_to_rollback: list[str] | None = None,
|
|
10
9
|
):
|
|
11
10
|
self.blueprints_to_rollback = blueprints_to_rollback
|
|
12
|
-
self.pages_to_rollback = pages_to_rollback
|
|
13
11
|
self.errors = errors
|
|
14
12
|
super().__init__("Aborting defaults creation")
|
|
15
13
|
|
port_ocean/helpers/retry.py
CHANGED
|
@@ -55,14 +55,14 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
55
55
|
HTTPStatus.GATEWAY_TIMEOUT,
|
|
56
56
|
]
|
|
57
57
|
)
|
|
58
|
-
|
|
58
|
+
MAX_BACKOFF_WAIT_IN_SECONDS = 60
|
|
59
59
|
|
|
60
60
|
def __init__(
|
|
61
61
|
self,
|
|
62
62
|
wrapped_transport: Union[httpx.BaseTransport, httpx.AsyncBaseTransport],
|
|
63
63
|
max_attempts: int = 10,
|
|
64
|
-
max_backoff_wait: float =
|
|
65
|
-
|
|
64
|
+
max_backoff_wait: float = MAX_BACKOFF_WAIT_IN_SECONDS,
|
|
65
|
+
base_delay: float = 0.1,
|
|
66
66
|
jitter_ratio: float = 0.1,
|
|
67
67
|
respect_retry_after_header: bool = True,
|
|
68
68
|
retryable_methods: Iterable[str] | None = None,
|
|
@@ -81,7 +81,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
81
81
|
max_backoff_wait (float, optional):
|
|
82
82
|
The maximum amount of time (in seconds) to wait before retrying a request.
|
|
83
83
|
Defaults to 60.
|
|
84
|
-
|
|
84
|
+
base_delay (float, optional):
|
|
85
85
|
The factor by which the waiting time will be multiplied in each retry attempt.
|
|
86
86
|
Defaults to 0.1.
|
|
87
87
|
jitter_ratio (float, optional):
|
|
@@ -105,7 +105,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
105
105
|
)
|
|
106
106
|
|
|
107
107
|
self._max_attempts = max_attempts
|
|
108
|
-
self.
|
|
108
|
+
self._base_delay = base_delay
|
|
109
109
|
self._respect_retry_after_header = respect_retry_after_header
|
|
110
110
|
self._retryable_methods = (
|
|
111
111
|
frozenset(retryable_methods)
|
|
@@ -132,13 +132,18 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
132
132
|
httpx.Response: The response received.
|
|
133
133
|
|
|
134
134
|
"""
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
135
|
+
try:
|
|
136
|
+
transport: httpx.BaseTransport = self._wrapped_transport # type: ignore
|
|
137
|
+
if self._is_retryable_method(request):
|
|
138
|
+
send_method = partial(transport.handle_request)
|
|
139
|
+
response = self._retry_operation(request, send_method)
|
|
140
|
+
else:
|
|
141
|
+
response = transport.handle_request(request)
|
|
142
|
+
return response
|
|
143
|
+
except Exception as e:
|
|
144
|
+
if not self._is_retryable_method(request) and self._logger is not None:
|
|
145
|
+
self._logger.exception(f"{repr(e)} - {request.url}", exc_info=e)
|
|
146
|
+
raise e
|
|
142
147
|
|
|
143
148
|
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
144
149
|
"""Sends an HTTP request, possibly with retries.
|
|
@@ -150,13 +155,19 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
150
155
|
The response.
|
|
151
156
|
|
|
152
157
|
"""
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
158
|
+
try:
|
|
159
|
+
transport: httpx.AsyncBaseTransport = self._wrapped_transport # type: ignore
|
|
160
|
+
if self._is_retryable_method(request):
|
|
161
|
+
send_method = partial(transport.handle_async_request)
|
|
162
|
+
response = await self._retry_operation_async(request, send_method)
|
|
163
|
+
else:
|
|
164
|
+
response = await transport.handle_async_request(request)
|
|
165
|
+
return response
|
|
166
|
+
except Exception as e:
|
|
167
|
+
# Retyable methods are logged via _log_error
|
|
168
|
+
if not self._is_retryable_method(request) and self._logger is not None:
|
|
169
|
+
self._logger.exception(f"{repr(e)} - {request.url}", exc_info=e)
|
|
170
|
+
raise e
|
|
160
171
|
|
|
161
172
|
async def aclose(self) -> None:
|
|
162
173
|
"""
|
|
@@ -179,12 +190,35 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
179
190
|
transport.close()
|
|
180
191
|
|
|
181
192
|
def _is_retryable_method(self, request: httpx.Request) -> bool:
|
|
182
|
-
return request.method in self._retryable_methods
|
|
193
|
+
return request.method in self._retryable_methods or request.extensions.get(
|
|
194
|
+
"retryable", False
|
|
195
|
+
)
|
|
183
196
|
|
|
184
197
|
def _should_retry(self, response: httpx.Response) -> bool:
|
|
185
198
|
return response.status_code in self._retry_status_codes
|
|
186
199
|
|
|
187
|
-
def
|
|
200
|
+
def _log_error(
|
|
201
|
+
self,
|
|
202
|
+
request: httpx.Request,
|
|
203
|
+
error: Exception | None,
|
|
204
|
+
) -> None:
|
|
205
|
+
if not self._logger:
|
|
206
|
+
return
|
|
207
|
+
|
|
208
|
+
if isinstance(error, httpx.ConnectTimeout):
|
|
209
|
+
self._logger.error(
|
|
210
|
+
f"Request {request.method} {request.url} failed to connect: {str(error)}"
|
|
211
|
+
)
|
|
212
|
+
elif isinstance(error, httpx.TimeoutException):
|
|
213
|
+
self._logger.error(
|
|
214
|
+
f"Request {request.method} {request.url} failed with a timeout exception: {str(error)}"
|
|
215
|
+
)
|
|
216
|
+
elif isinstance(error, httpx.HTTPError):
|
|
217
|
+
self._logger.error(
|
|
218
|
+
f"Request {request.method} {request.url} failed with an HTTP error: {str(error)}"
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
def _log_before_retry(
|
|
188
222
|
self,
|
|
189
223
|
request: httpx.Request,
|
|
190
224
|
sleep_time: float,
|
|
@@ -232,7 +266,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
232
266
|
except ValueError:
|
|
233
267
|
pass
|
|
234
268
|
|
|
235
|
-
backoff = self.
|
|
269
|
+
backoff = self._base_delay * (2 ** (attempts_made - 1))
|
|
236
270
|
jitter = (backoff * self._jitter_ratio) * random.choice([1, -1])
|
|
237
271
|
total_backoff = backoff + jitter
|
|
238
272
|
return min(total_backoff, self._max_backoff_wait)
|
|
@@ -249,7 +283,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
249
283
|
while True:
|
|
250
284
|
if attempts_made > 0:
|
|
251
285
|
sleep_time = self._calculate_sleep(attempts_made, {})
|
|
252
|
-
self.
|
|
286
|
+
self._log_before_retry(request, sleep_time, response, error)
|
|
253
287
|
await asyncio.sleep(sleep_time)
|
|
254
288
|
|
|
255
289
|
error = None
|
|
@@ -262,9 +296,25 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
262
296
|
):
|
|
263
297
|
return response
|
|
264
298
|
await response.aclose()
|
|
299
|
+
except httpx.ConnectTimeout as e:
|
|
300
|
+
error = e
|
|
301
|
+
if remaining_attempts < 1:
|
|
302
|
+
self._log_error(request, error)
|
|
303
|
+
raise
|
|
304
|
+
except httpx.ReadTimeout as e:
|
|
305
|
+
error = e
|
|
306
|
+
if remaining_attempts < 1:
|
|
307
|
+
self._log_error(request, error)
|
|
308
|
+
raise
|
|
309
|
+
except httpx.TimeoutException as e:
|
|
310
|
+
error = e
|
|
311
|
+
if remaining_attempts < 1:
|
|
312
|
+
self._log_error(request, error)
|
|
313
|
+
raise
|
|
265
314
|
except httpx.HTTPError as e:
|
|
266
315
|
error = e
|
|
267
316
|
if remaining_attempts < 1:
|
|
317
|
+
self._log_error(request, error)
|
|
268
318
|
raise
|
|
269
319
|
attempts_made += 1
|
|
270
320
|
remaining_attempts -= 1
|
|
@@ -281,7 +331,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
281
331
|
while True:
|
|
282
332
|
if attempts_made > 0:
|
|
283
333
|
sleep_time = self._calculate_sleep(attempts_made, {})
|
|
284
|
-
self.
|
|
334
|
+
self._log_before_retry(request, sleep_time, response, error)
|
|
285
335
|
time.sleep(sleep_time)
|
|
286
336
|
|
|
287
337
|
error = None
|
|
@@ -292,9 +342,20 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
|
292
342
|
if remaining_attempts < 1 or not self._should_retry(response):
|
|
293
343
|
return response
|
|
294
344
|
response.close()
|
|
345
|
+
except httpx.ConnectTimeout as e:
|
|
346
|
+
error = e
|
|
347
|
+
if remaining_attempts < 1:
|
|
348
|
+
self._log_error(request, error)
|
|
349
|
+
raise
|
|
350
|
+
except httpx.TimeoutException as e:
|
|
351
|
+
error = e
|
|
352
|
+
if remaining_attempts < 1:
|
|
353
|
+
self._log_error(request, error)
|
|
354
|
+
raise
|
|
295
355
|
except httpx.HTTPError as e:
|
|
296
356
|
error = e
|
|
297
357
|
if remaining_attempts < 1:
|
|
358
|
+
self._log_error(request, error)
|
|
298
359
|
raise
|
|
299
360
|
attempts_made += 1
|
|
300
361
|
remaining_attempts -= 1
|
port_ocean/log/handlers.py
CHANGED
|
@@ -11,16 +11,22 @@ from loguru import logger
|
|
|
11
11
|
|
|
12
12
|
from port_ocean import Ocean
|
|
13
13
|
from port_ocean.context.ocean import ocean
|
|
14
|
+
from copy import deepcopy
|
|
15
|
+
from traceback import format_exception
|
|
14
16
|
|
|
15
17
|
|
|
16
18
|
def _serialize_record(record: logging.LogRecord) -> dict[str, Any]:
|
|
19
|
+
extra = {**deepcopy(record.__dict__["extra"])}
|
|
20
|
+
if isinstance(extra.get("exc_info"), Exception):
|
|
21
|
+
serialized_exception = "".join(format_exception(extra.get("exc_info")))
|
|
22
|
+
extra["exc_info"] = serialized_exception
|
|
17
23
|
return {
|
|
18
24
|
"message": record.msg,
|
|
19
25
|
"level": record.levelname,
|
|
20
26
|
"timestamp": datetime.utcfromtimestamp(record.created).strftime(
|
|
21
27
|
"%Y-%m-%dT%H:%M:%S.%fZ"
|
|
22
28
|
),
|
|
23
|
-
"extra":
|
|
29
|
+
"extra": extra,
|
|
24
30
|
}
|
|
25
31
|
|
|
26
32
|
|
|
@@ -37,6 +43,7 @@ class HTTPMemoryHandler(MemoryHandler):
|
|
|
37
43
|
self.flush_size = flush_size
|
|
38
44
|
self.last_flush_time = time.time()
|
|
39
45
|
self._serialized_buffer: list[dict[str, Any]] = []
|
|
46
|
+
self._thread_pool: list[threading.Thread] = []
|
|
40
47
|
|
|
41
48
|
@property
|
|
42
49
|
def ocean(self) -> Ocean | None:
|
|
@@ -46,6 +53,7 @@ class HTTPMemoryHandler(MemoryHandler):
|
|
|
46
53
|
return None
|
|
47
54
|
|
|
48
55
|
def emit(self, record: logging.LogRecord) -> None:
|
|
56
|
+
|
|
49
57
|
self._serialized_buffer.append(_serialize_record(record))
|
|
50
58
|
super().emit(record)
|
|
51
59
|
|
|
@@ -61,6 +69,11 @@ class HTTPMemoryHandler(MemoryHandler):
|
|
|
61
69
|
return True
|
|
62
70
|
return False
|
|
63
71
|
|
|
72
|
+
def wait_for_lingering_threads(self) -> None:
|
|
73
|
+
for thread in self._thread_pool:
|
|
74
|
+
if thread.is_alive():
|
|
75
|
+
thread.join()
|
|
76
|
+
|
|
64
77
|
def flush(self) -> None:
|
|
65
78
|
if self.ocean is None or not self.buffer:
|
|
66
79
|
return
|
|
@@ -70,13 +83,21 @@ class HTTPMemoryHandler(MemoryHandler):
|
|
|
70
83
|
loop.run_until_complete(self.send_logs(_ocean, logs_to_send))
|
|
71
84
|
loop.close()
|
|
72
85
|
|
|
86
|
+
def clear_thread_pool() -> None:
|
|
87
|
+
for thread in self._thread_pool:
|
|
88
|
+
if not thread.is_alive():
|
|
89
|
+
self._thread_pool.remove(thread)
|
|
90
|
+
|
|
73
91
|
self.acquire()
|
|
74
92
|
logs = list(self._serialized_buffer)
|
|
75
93
|
if logs:
|
|
76
94
|
self.buffer.clear()
|
|
77
95
|
self._serialized_buffer.clear()
|
|
78
96
|
self.last_flush_time = time.time()
|
|
79
|
-
|
|
97
|
+
clear_thread_pool()
|
|
98
|
+
thread = threading.Thread(target=_wrap_event_loop, args=(self.ocean, logs))
|
|
99
|
+
thread.start()
|
|
100
|
+
self._thread_pool.append(thread)
|
|
80
101
|
self.release()
|
|
81
102
|
|
|
82
103
|
async def send_logs(
|
port_ocean/log/logger_setup.py
CHANGED
|
@@ -9,6 +9,7 @@ from loguru import logger
|
|
|
9
9
|
from port_ocean.config.settings import LogLevelType
|
|
10
10
|
from port_ocean.log.handlers import HTTPMemoryHandler
|
|
11
11
|
from port_ocean.log.sensetive import sensitive_log_filter
|
|
12
|
+
from port_ocean.utils.signal import signal_handler
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
def setup_logger(level: LogLevelType, enable_http_handler: bool) -> None:
|
|
@@ -35,6 +36,7 @@ def _stdout_loguru_handler(level: LogLevelType) -> None:
|
|
|
35
36
|
diagnose=False, # hide variable values in log backtrace
|
|
36
37
|
filter=sensitive_log_filter.create_filter(),
|
|
37
38
|
)
|
|
39
|
+
logger.configure(patcher=exception_deserializer)
|
|
38
40
|
|
|
39
41
|
|
|
40
42
|
def _http_loguru_handler(level: LogLevelType) -> None:
|
|
@@ -50,8 +52,13 @@ def _http_loguru_handler(level: LogLevelType) -> None:
|
|
|
50
52
|
enqueue=True, # process logs in background
|
|
51
53
|
filter=sensitive_log_filter.create_filter(full_hide=True),
|
|
52
54
|
)
|
|
55
|
+
logger.configure(patcher=exception_deserializer)
|
|
53
56
|
|
|
54
|
-
|
|
57
|
+
http_memory_handler = HTTPMemoryHandler()
|
|
58
|
+
signal_handler.register(http_memory_handler.wait_for_lingering_threads)
|
|
59
|
+
signal_handler.register(http_memory_handler.flush)
|
|
60
|
+
|
|
61
|
+
queue_listener = QueueListener(queue, http_memory_handler)
|
|
55
62
|
queue_listener.start()
|
|
56
63
|
|
|
57
64
|
|
port_ocean/log/sensetive.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import re
|
|
2
|
-
from typing import Callable, TYPE_CHECKING
|
|
2
|
+
from typing import Any, Callable, TYPE_CHECKING
|
|
3
3
|
|
|
4
4
|
if TYPE_CHECKING:
|
|
5
5
|
from loguru import Record
|
|
@@ -21,7 +21,7 @@ secret_patterns = {
|
|
|
21
21
|
"GitHub": r"[g|G][i|I][t|T][h|H][u|U][b|B].*['|\"][0-9a-zA-Z]{35,40}['|\"]",
|
|
22
22
|
"Google Cloud Platform API Key": r"AIza[0-9A-Za-z\\-_]{35}",
|
|
23
23
|
"Google Cloud Platform OAuth": r"[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com",
|
|
24
|
-
"Google (GCP) Service-account":
|
|
24
|
+
"Google (GCP) Service-account": f'"type":{" "}"service_account"',
|
|
25
25
|
"Google OAuth Access Token": r"ya29\\.[0-9A-Za-z\\-_]+",
|
|
26
26
|
"Connection String": r"[a-zA-Z]+:\/\/[^/\s]+:[^/\s]+@[^/\s]+\/[^/\s]+",
|
|
27
27
|
}
|
|
@@ -35,16 +35,31 @@ class SensitiveLogFilter:
|
|
|
35
35
|
[re.compile(re.escape(token.strip())) for token in tokens if token.strip()]
|
|
36
36
|
)
|
|
37
37
|
|
|
38
|
+
def mask_string(self, string: str, full_hide: bool = False) -> str:
|
|
39
|
+
masked_string = string
|
|
40
|
+
for pattern in self.compiled_patterns:
|
|
41
|
+
replace: Callable[[re.Match[str]], str] | str = (
|
|
42
|
+
"[REDACTED]"
|
|
43
|
+
if full_hide
|
|
44
|
+
else lambda match: match.group()[:6] + "[REDACTED]"
|
|
45
|
+
)
|
|
46
|
+
masked_string = pattern.sub(replace, masked_string)
|
|
47
|
+
return masked_string
|
|
48
|
+
|
|
49
|
+
def mask_object(self, obj: Any, full_hide: bool = False) -> Any:
|
|
50
|
+
if isinstance(obj, str):
|
|
51
|
+
return self.mask_string(obj, full_hide)
|
|
52
|
+
if isinstance(obj, list):
|
|
53
|
+
return [self.mask_object(o, full_hide) for o in obj]
|
|
54
|
+
if isinstance(obj, dict):
|
|
55
|
+
for k, v in obj.items():
|
|
56
|
+
obj[k] = self.mask_object(v, full_hide)
|
|
57
|
+
|
|
58
|
+
return obj
|
|
59
|
+
|
|
38
60
|
def create_filter(self, full_hide: bool = False) -> Callable[["Record"], bool]:
|
|
39
61
|
def _filter(record: "Record") -> bool:
|
|
40
|
-
|
|
41
|
-
replace: Callable[[re.Match[str]], str] | str = (
|
|
42
|
-
"[REDACTED]"
|
|
43
|
-
if full_hide
|
|
44
|
-
else lambda match: match.group()[:6] + "[REDACTED]"
|
|
45
|
-
)
|
|
46
|
-
record["message"] = pattern.sub(replace, record["message"])
|
|
47
|
-
|
|
62
|
+
record["message"] = self.mask_string(record["message"], full_hide)
|
|
48
63
|
return True
|
|
49
64
|
|
|
50
65
|
return _filter
|
port_ocean/middlewares.py
CHANGED
|
@@ -52,7 +52,14 @@ async def request_handler(
|
|
|
52
52
|
request_id = generate_uuid()
|
|
53
53
|
|
|
54
54
|
with logger.contextualize(request_id=request_id):
|
|
55
|
-
|
|
55
|
+
log_level = (
|
|
56
|
+
"DEBUG"
|
|
57
|
+
if request.url.path == "/docs" or request.url.path == "/openapi.json"
|
|
58
|
+
else "INFO"
|
|
59
|
+
)
|
|
60
|
+
logger.bind(url=str(request.url), method=request.method).log(
|
|
61
|
+
log_level, f"Request to {request.url.path} started"
|
|
62
|
+
)
|
|
56
63
|
response = await _handle_silently(call_next, request)
|
|
57
64
|
|
|
58
65
|
end_time = get_time(seconds_precision=False)
|
|
@@ -61,5 +68,6 @@ async def request_handler(
|
|
|
61
68
|
response.headers["X-Process-Time"] = str(time_elapsed)
|
|
62
69
|
logger.bind(
|
|
63
70
|
time_elapsed=time_elapsed, response_status=response.status_code
|
|
64
|
-
).
|
|
71
|
+
).log(log_level, f"Request to {request.url.path} ended")
|
|
72
|
+
|
|
65
73
|
return response
|
port_ocean/ocean.py
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import sys
|
|
3
3
|
import threading
|
|
4
|
-
from
|
|
4
|
+
from contextlib import asynccontextmanager
|
|
5
|
+
from typing import Callable, Any, Dict, AsyncIterator, Type
|
|
5
6
|
|
|
6
7
|
from fastapi import FastAPI, APIRouter
|
|
7
8
|
from loguru import logger
|
|
8
9
|
from pydantic import BaseModel
|
|
9
10
|
from starlette.types import Scope, Receive, Send
|
|
10
11
|
|
|
12
|
+
from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
|
|
11
13
|
from port_ocean.clients.port.client import PortClient
|
|
12
14
|
from port_ocean.config.settings import (
|
|
13
15
|
IntegrationConfiguration,
|
|
@@ -21,8 +23,9 @@ from port_ocean.core.integrations.base import BaseIntegration
|
|
|
21
23
|
from port_ocean.log.sensetive import sensitive_log_filter
|
|
22
24
|
from port_ocean.middlewares import request_handler
|
|
23
25
|
from port_ocean.utils.repeat import repeat_every
|
|
24
|
-
from port_ocean.utils.signal import
|
|
26
|
+
from port_ocean.utils.signal import signal_handler
|
|
25
27
|
from port_ocean.version import __integration_version__
|
|
28
|
+
from port_ocean.utils.misc import IntegrationStateStatus
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
class Ocean:
|
|
@@ -31,7 +34,7 @@ class Ocean:
|
|
|
31
34
|
app: FastAPI | None = None,
|
|
32
35
|
integration_class: Callable[[PortOceanContext], BaseIntegration] | None = None,
|
|
33
36
|
integration_router: APIRouter | None = None,
|
|
34
|
-
config_factory:
|
|
37
|
+
config_factory: Type[BaseModel] | None = None,
|
|
35
38
|
config_override: Dict[str, Any] | None = None,
|
|
36
39
|
):
|
|
37
40
|
initialize_port_ocean_context(self)
|
|
@@ -39,16 +42,11 @@ class Ocean:
|
|
|
39
42
|
self.fast_api_app.middleware("http")(request_handler)
|
|
40
43
|
|
|
41
44
|
self.config = IntegrationConfiguration(
|
|
42
|
-
|
|
45
|
+
# type: ignore
|
|
46
|
+
_integration_config_model=config_factory,
|
|
47
|
+
**(config_override or {}),
|
|
43
48
|
)
|
|
44
49
|
|
|
45
|
-
if config_factory:
|
|
46
|
-
raw_config = (
|
|
47
|
-
self.config.integration.config
|
|
48
|
-
if isinstance(self.config.integration.config, dict)
|
|
49
|
-
else self.config.integration.config.dict()
|
|
50
|
-
)
|
|
51
|
-
self.config.integration.config = config_factory(**raw_config)
|
|
52
50
|
# add the integration sensitive configuration to the sensitive patterns to mask out
|
|
53
51
|
sensitive_log_filter.hide_sensitive_strings(
|
|
54
52
|
*self.config.get_sensitive_fields_data()
|
|
@@ -67,40 +65,75 @@ class Ocean:
|
|
|
67
65
|
integration_class(ocean) if integration_class else BaseIntegration(ocean)
|
|
68
66
|
)
|
|
69
67
|
|
|
68
|
+
self.resync_state_updater = ResyncStateUpdater(
|
|
69
|
+
self.port_client, self.config.scheduled_resync_interval
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
self.app_initialized = False
|
|
73
|
+
|
|
74
|
+
def is_saas(self) -> bool:
|
|
75
|
+
return self.config.runtime.is_saas_runtime
|
|
76
|
+
|
|
70
77
|
async def _setup_scheduled_resync(
|
|
71
78
|
self,
|
|
72
79
|
) -> None:
|
|
73
|
-
def execute_resync_all() -> None:
|
|
74
|
-
|
|
75
|
-
asyncio.set_event_loop(loop)
|
|
76
|
-
|
|
80
|
+
async def execute_resync_all() -> None:
|
|
81
|
+
await self.resync_state_updater.update_before_resync()
|
|
77
82
|
logger.info("Starting a new scheduled resync")
|
|
78
|
-
|
|
79
|
-
|
|
83
|
+
try:
|
|
84
|
+
await self.integration.sync_raw_all()
|
|
85
|
+
await self.resync_state_updater.update_after_resync()
|
|
86
|
+
except asyncio.CancelledError:
|
|
87
|
+
logger.warning(
|
|
88
|
+
"resync was cancelled by the scheduled resync, skipping state update"
|
|
89
|
+
)
|
|
90
|
+
except Exception as e:
|
|
91
|
+
await self.resync_state_updater.update_after_resync(
|
|
92
|
+
IntegrationStateStatus.Failed
|
|
93
|
+
)
|
|
94
|
+
raise e
|
|
80
95
|
|
|
81
96
|
interval = self.config.scheduled_resync_interval
|
|
97
|
+
loop = asyncio.get_event_loop()
|
|
82
98
|
if interval is not None:
|
|
83
99
|
logger.info(
|
|
84
|
-
f"Setting up scheduled resync, the integration will automatically perform a full resync every {interval} minutes)"
|
|
100
|
+
f"Setting up scheduled resync, the integration will automatically perform a full resync every {interval} minutes)",
|
|
101
|
+
scheduled_interval=interval,
|
|
85
102
|
)
|
|
86
103
|
repeated_function = repeat_every(
|
|
87
104
|
seconds=interval * 60,
|
|
88
105
|
# Not running the resync immediately because the event listener should run resync on startup
|
|
89
106
|
wait_first=True,
|
|
90
|
-
)(
|
|
107
|
+
)(
|
|
108
|
+
lambda: threading.Thread(
|
|
109
|
+
target=lambda: asyncio.run_coroutine_threadsafe(
|
|
110
|
+
execute_resync_all(), loop
|
|
111
|
+
)
|
|
112
|
+
).start()
|
|
113
|
+
)
|
|
91
114
|
await repeated_function()
|
|
92
115
|
|
|
93
|
-
|
|
116
|
+
def initialize_app(self) -> None:
|
|
94
117
|
self.fast_api_app.include_router(self.integration_router, prefix="/integration")
|
|
95
118
|
|
|
96
|
-
@
|
|
97
|
-
async def
|
|
98
|
-
init_signal_handler()
|
|
119
|
+
@asynccontextmanager
|
|
120
|
+
async def lifecycle(_: FastAPI) -> AsyncIterator[None]:
|
|
99
121
|
try:
|
|
100
122
|
await self.integration.start()
|
|
101
123
|
await self._setup_scheduled_resync()
|
|
124
|
+
yield None
|
|
102
125
|
except Exception:
|
|
103
|
-
logger.exception("
|
|
126
|
+
logger.exception("Integration had a fatal error. Shutting down.")
|
|
127
|
+
logger.complete()
|
|
104
128
|
sys.exit("Server stopped")
|
|
129
|
+
finally:
|
|
130
|
+
signal_handler.exit()
|
|
131
|
+
|
|
132
|
+
self.fast_api_app.router.lifespan_context = lifecycle
|
|
133
|
+
self.app_initialized = True
|
|
134
|
+
|
|
135
|
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
|
136
|
+
if not self.app_initialized:
|
|
137
|
+
self.initialize_app()
|
|
105
138
|
|
|
106
139
|
await self.fast_api_app(scope, receive, send)
|