port-ocean 0.27.10__py3-none-any.whl → 0.28.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- port_ocean/core/integrations/mixins/sync_raw.py +10 -10
- port_ocean/helpers/async_client.py +7 -8
- port_ocean/helpers/retry.py +162 -72
- port_ocean/tests/helpers/test_retry.py +309 -0
- {port_ocean-0.27.10.dist-info → port_ocean-0.28.1.dist-info}/METADATA +1 -1
- {port_ocean-0.27.10.dist-info → port_ocean-0.28.1.dist-info}/RECORD +9 -8
- {port_ocean-0.27.10.dist-info → port_ocean-0.28.1.dist-info}/LICENSE.md +0 -0
- {port_ocean-0.27.10.dist-info → port_ocean-0.28.1.dist-info}/WHEEL +0 -0
- {port_ocean-0.27.10.dist-info → port_ocean-0.28.1.dist-info}/entry_points.txt +0 -0
@@ -455,16 +455,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
|
|
455
455
|
],
|
456
456
|
value=number_of_transformed_entities,
|
457
457
|
)
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
458
|
+
if number_of_raw_results > number_of_transformed_entities :
|
459
|
+
ocean.metrics.inc_metric(
|
460
|
+
name=MetricType.OBJECT_COUNT_NAME,
|
461
|
+
labels=[
|
462
|
+
ocean.metrics.current_resource_kind(),
|
463
|
+
MetricPhase.TRANSFORM,
|
464
|
+
MetricPhase.TransformResult.FILTERED_OUT,
|
465
|
+
],
|
466
|
+
value=number_of_raw_results - number_of_transformed_entities,
|
467
|
+
)
|
468
468
|
|
469
469
|
return passed_entities, errors
|
470
470
|
|
@@ -3,7 +3,7 @@ from typing import Any, Type
|
|
3
3
|
import httpx
|
4
4
|
from loguru import logger
|
5
5
|
|
6
|
-
from port_ocean.helpers.retry import RetryTransport
|
6
|
+
from port_ocean.helpers.retry import RetryTransport, RetryConfig
|
7
7
|
from port_ocean.helpers.stream import Stream
|
8
8
|
|
9
9
|
|
@@ -18,10 +18,12 @@ class OceanAsyncClient(httpx.AsyncClient):
|
|
18
18
|
self,
|
19
19
|
transport_class: Type[RetryTransport] = RetryTransport,
|
20
20
|
transport_kwargs: dict[str, Any] | None = None,
|
21
|
+
retry_config: RetryConfig | None = None,
|
21
22
|
**kwargs: Any,
|
22
23
|
):
|
23
24
|
self._transport_kwargs = transport_kwargs
|
24
25
|
self._transport_class = transport_class
|
26
|
+
self._retry_config = retry_config
|
25
27
|
super().__init__(**kwargs)
|
26
28
|
|
27
29
|
def _init_transport( # type: ignore[override]
|
@@ -33,9 +35,8 @@ class OceanAsyncClient(httpx.AsyncClient):
|
|
33
35
|
return super()._init_transport(transport=transport, **kwargs)
|
34
36
|
|
35
37
|
return self._transport_class(
|
36
|
-
wrapped_transport=httpx.AsyncHTTPTransport(
|
37
|
-
|
38
|
-
),
|
38
|
+
wrapped_transport=httpx.AsyncHTTPTransport(**kwargs),
|
39
|
+
retry_config=self._retry_config,
|
39
40
|
logger=logger,
|
40
41
|
**(self._transport_kwargs or {}),
|
41
42
|
)
|
@@ -44,10 +45,8 @@ class OceanAsyncClient(httpx.AsyncClient):
|
|
44
45
|
self, proxy: httpx.Proxy, **kwargs: Any
|
45
46
|
) -> httpx.AsyncBaseTransport:
|
46
47
|
return self._transport_class(
|
47
|
-
wrapped_transport=httpx.AsyncHTTPTransport(
|
48
|
-
|
49
|
-
**kwargs,
|
50
|
-
),
|
48
|
+
wrapped_transport=httpx.AsyncHTTPTransport(proxy=proxy, **kwargs),
|
49
|
+
retry_config=self._retry_config,
|
51
50
|
logger=logger,
|
52
51
|
**(self._transport_kwargs or {}),
|
53
52
|
)
|
port_ocean/helpers/retry.py
CHANGED
@@ -4,12 +4,24 @@ import time
|
|
4
4
|
from datetime import datetime
|
5
5
|
from functools import partial
|
6
6
|
from http import HTTPStatus
|
7
|
-
from typing import
|
7
|
+
from typing import (
|
8
|
+
Any,
|
9
|
+
Callable,
|
10
|
+
Coroutine,
|
11
|
+
Iterable,
|
12
|
+
Mapping,
|
13
|
+
Union,
|
14
|
+
cast,
|
15
|
+
Optional,
|
16
|
+
List,
|
17
|
+
)
|
8
18
|
import httpx
|
9
19
|
from dateutil.parser import isoparse
|
10
20
|
import logging
|
11
21
|
|
22
|
+
MAX_BACKOFF_WAIT_IN_SECONDS = 60
|
12
23
|
_ON_RETRY_CALLBACK: Callable[[httpx.Request], httpx.Request] | None = None
|
24
|
+
_RETRY_CONFIG_CALLBACK: Callable[[], "RetryConfig"] | None = None
|
13
25
|
|
14
26
|
|
15
27
|
def register_on_retry_callback(
|
@@ -19,6 +31,92 @@ def register_on_retry_callback(
|
|
19
31
|
_ON_RETRY_CALLBACK = _on_retry_callback
|
20
32
|
|
21
33
|
|
34
|
+
def register_retry_config_callback(
|
35
|
+
retry_config_callback: Callable[[], "RetryConfig"]
|
36
|
+
) -> None:
|
37
|
+
"""Register a callback function that returns a RetryConfig instance.
|
38
|
+
|
39
|
+
The callback will be called when a RetryTransport needs to be created.
|
40
|
+
|
41
|
+
Args:
|
42
|
+
retry_config_callback: A function that returns a RetryConfig instance
|
43
|
+
"""
|
44
|
+
global _RETRY_CONFIG_CALLBACK
|
45
|
+
_RETRY_CONFIG_CALLBACK = retry_config_callback
|
46
|
+
|
47
|
+
|
48
|
+
class RetryConfig:
|
49
|
+
"""Configuration class for retry behavior that can be customized per integration."""
|
50
|
+
|
51
|
+
def __init__(
|
52
|
+
self,
|
53
|
+
max_attempts: int = 10,
|
54
|
+
max_backoff_wait: float = MAX_BACKOFF_WAIT_IN_SECONDS,
|
55
|
+
base_delay: float = 0.1,
|
56
|
+
jitter_ratio: float = 0.1,
|
57
|
+
respect_retry_after_header: bool = True,
|
58
|
+
retryable_methods: Optional[Iterable[str]] = None,
|
59
|
+
retry_status_codes: Optional[Iterable[int]] = None,
|
60
|
+
retry_after_headers: Optional[List[str]] = None,
|
61
|
+
additional_retry_status_codes: Optional[Iterable[int]] = None,
|
62
|
+
):
|
63
|
+
"""
|
64
|
+
Initialize retry configuration.
|
65
|
+
|
66
|
+
Args:
|
67
|
+
max_attempts: Maximum number of retry attempts
|
68
|
+
max_backoff_wait: Maximum backoff wait time in seconds
|
69
|
+
base_delay: Base delay for exponential backoff
|
70
|
+
jitter_ratio: Jitter ratio for backoff (0-0.5)
|
71
|
+
respect_retry_after_header: Whether to respect Retry-After header
|
72
|
+
retryable_methods: HTTP methods that can be retried (overrides defaults if provided)
|
73
|
+
retry_status_codes: DEPRECATED - use additional_retry_status_codes instead
|
74
|
+
retry_after_headers: Custom headers to check for retry timing (e.g., ['X-RateLimit-Reset', 'Retry-After'])
|
75
|
+
additional_retry_status_codes: Additional status codes to retry (extends system defaults)
|
76
|
+
"""
|
77
|
+
self.max_attempts = max_attempts
|
78
|
+
self.max_backoff_wait = max_backoff_wait
|
79
|
+
self.base_delay = base_delay
|
80
|
+
self.jitter_ratio = jitter_ratio
|
81
|
+
self.respect_retry_after_header = respect_retry_after_header
|
82
|
+
|
83
|
+
# Default retryable methods - always include these unless explicitly overridden
|
84
|
+
default_methods = frozenset(
|
85
|
+
["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
|
86
|
+
)
|
87
|
+
self.retryable_methods = (
|
88
|
+
frozenset(retryable_methods) if retryable_methods else default_methods
|
89
|
+
)
|
90
|
+
|
91
|
+
# Default retry status codes - always include these for system reliability
|
92
|
+
default_status_codes = frozenset(
|
93
|
+
[
|
94
|
+
HTTPStatus.TOO_MANY_REQUESTS,
|
95
|
+
HTTPStatus.BAD_GATEWAY,
|
96
|
+
HTTPStatus.SERVICE_UNAVAILABLE,
|
97
|
+
HTTPStatus.GATEWAY_TIMEOUT,
|
98
|
+
HTTPStatus.UNAUTHORIZED,
|
99
|
+
HTTPStatus.BAD_REQUEST,
|
100
|
+
]
|
101
|
+
)
|
102
|
+
|
103
|
+
# Additional status codes to retry (extends defaults)
|
104
|
+
additional_codes = (
|
105
|
+
frozenset(additional_retry_status_codes)
|
106
|
+
if additional_retry_status_codes
|
107
|
+
else frozenset()
|
108
|
+
)
|
109
|
+
|
110
|
+
# Combine defaults with additional codes for extensibility
|
111
|
+
self.retry_status_codes = default_status_codes | additional_codes
|
112
|
+
self.retry_after_headers = retry_after_headers or ["Retry-After"]
|
113
|
+
|
114
|
+
if jitter_ratio < 0 or jitter_ratio > 0.5:
|
115
|
+
raise ValueError(
|
116
|
+
f"Jitter ratio should be between 0 and 0.5, actual {jitter_ratio}"
|
117
|
+
)
|
118
|
+
|
119
|
+
|
22
120
|
# Adapted from https://github.com/encode/httpx/issues/108#issuecomment-1434439481
|
23
121
|
class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
24
122
|
"""
|
@@ -41,32 +139,16 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
41
139
|
["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"].
|
42
140
|
retry_status_codes (Iterable[int], optional): The HTTP status codes that can be retried. Defaults to
|
43
141
|
[429, 502, 503, 504].
|
142
|
+
retry_config (RetryConfig, optional): Configuration for retry behavior. If not provided, uses defaults.
|
143
|
+
logger (Any, optional): The logger to use for logging retries.
|
44
144
|
|
45
145
|
Attributes:
|
46
146
|
_wrapped_transport (Union[httpx.BaseTransport, httpx.AsyncBaseTransport]): The underlying HTTP transport
|
47
147
|
being wrapped.
|
48
|
-
|
49
|
-
|
50
|
-
_respect_retry_after_header (bool): Whether to respect the Retry-After header in HTTP responses.
|
51
|
-
_retryable_methods (frozenset): The HTTP methods that can be retried.
|
52
|
-
_retry_status_codes (frozenset): The HTTP status codes that can be retried.
|
53
|
-
_jitter_ratio (float): The amount of jitter to add to the backoff time.
|
54
|
-
_max_backoff_wait (float): The maximum time to wait between retries in seconds.
|
148
|
+
_retry_config (RetryConfig): The retry configuration object.
|
149
|
+
_logger (Any): The logger to use for logging retries.
|
55
150
|
"""
|
56
151
|
|
57
|
-
RETRYABLE_METHODS = frozenset(["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"])
|
58
|
-
RETRYABLE_STATUS_CODES = frozenset(
|
59
|
-
[
|
60
|
-
HTTPStatus.TOO_MANY_REQUESTS,
|
61
|
-
HTTPStatus.BAD_GATEWAY,
|
62
|
-
HTTPStatus.SERVICE_UNAVAILABLE,
|
63
|
-
HTTPStatus.GATEWAY_TIMEOUT,
|
64
|
-
HTTPStatus.UNAUTHORIZED,
|
65
|
-
HTTPStatus.BAD_REQUEST,
|
66
|
-
]
|
67
|
-
)
|
68
|
-
MAX_BACKOFF_WAIT_IN_SECONDS = 60
|
69
|
-
|
70
152
|
def __init__(
|
71
153
|
self,
|
72
154
|
wrapped_transport: Union[httpx.BaseTransport, httpx.AsyncBaseTransport],
|
@@ -77,6 +159,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
77
159
|
respect_retry_after_header: bool = True,
|
78
160
|
retryable_methods: Iterable[str] | None = None,
|
79
161
|
retry_status_codes: Iterable[int] | None = None,
|
162
|
+
retry_config: Optional[RetryConfig] = None,
|
80
163
|
logger: Any | None = None,
|
81
164
|
) -> None:
|
82
165
|
"""
|
@@ -106,29 +189,27 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
106
189
|
retry_status_codes (Iterable[int], optional):
|
107
190
|
The HTTP status codes that can be retried.
|
108
191
|
Defaults to [429, 502, 503, 504].
|
192
|
+
retry_config (RetryConfig, optional):
|
193
|
+
Configuration for retry behavior. If not provided, uses default configuration.
|
109
194
|
logger (Any): The logger to use for logging retries.
|
110
195
|
"""
|
111
196
|
self._wrapped_transport = wrapped_transport
|
112
|
-
|
113
|
-
|
114
|
-
|
197
|
+
|
198
|
+
if retry_config is not None:
|
199
|
+
self._retry_config = retry_config
|
200
|
+
elif _RETRY_CONFIG_CALLBACK is not None:
|
201
|
+
self._retry_config = _RETRY_CONFIG_CALLBACK()
|
202
|
+
else:
|
203
|
+
self._retry_config = RetryConfig(
|
204
|
+
max_attempts=max_attempts,
|
205
|
+
max_backoff_wait=max_backoff_wait,
|
206
|
+
base_delay=base_delay,
|
207
|
+
jitter_ratio=jitter_ratio,
|
208
|
+
respect_retry_after_header=respect_retry_after_header,
|
209
|
+
retryable_methods=retryable_methods,
|
210
|
+
retry_status_codes=retry_status_codes,
|
115
211
|
)
|
116
212
|
|
117
|
-
self._max_attempts = max_attempts
|
118
|
-
self._base_delay = base_delay
|
119
|
-
self._respect_retry_after_header = respect_retry_after_header
|
120
|
-
self._retryable_methods = (
|
121
|
-
frozenset(retryable_methods)
|
122
|
-
if retryable_methods
|
123
|
-
else self.RETRYABLE_METHODS
|
124
|
-
)
|
125
|
-
self._retry_status_codes = (
|
126
|
-
frozenset(retry_status_codes)
|
127
|
-
if retry_status_codes
|
128
|
-
else self.RETRYABLE_STATUS_CODES
|
129
|
-
)
|
130
|
-
self._jitter_ratio = jitter_ratio
|
131
|
-
self._max_backoff_wait = max_backoff_wait
|
132
213
|
self._logger = logger
|
133
214
|
|
134
215
|
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
@@ -206,12 +287,13 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
206
287
|
transport.close()
|
207
288
|
|
208
289
|
def _is_retryable_method(self, request: httpx.Request) -> bool:
|
209
|
-
return
|
210
|
-
|
290
|
+
return (
|
291
|
+
request.method in self._retry_config.retryable_methods
|
292
|
+
or request.extensions.get("retryable", False)
|
211
293
|
)
|
212
294
|
|
213
295
|
def _should_retry(self, response: httpx.Response) -> bool:
|
214
|
-
return response.status_code in self.
|
296
|
+
return response.status_code in self._retry_config.retry_status_codes
|
215
297
|
|
216
298
|
def _log_error(
|
217
299
|
self,
|
@@ -314,46 +396,54 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
314
396
|
)
|
315
397
|
|
316
398
|
async def _should_retry_async(self, response: httpx.Response) -> bool:
|
317
|
-
return response.status_code in self.
|
399
|
+
return response.status_code in self._retry_config.retry_status_codes
|
318
400
|
|
319
401
|
def _calculate_sleep(
|
320
402
|
self, attempts_made: int, headers: Union[httpx.Headers, Mapping[str, str]]
|
321
403
|
) -> float:
|
322
|
-
# Retry-After
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
#
|
331
|
-
|
332
|
-
|
333
|
-
if retry_after_header.isdigit():
|
334
|
-
return float(retry_after_header)
|
335
|
-
|
336
|
-
try:
|
337
|
-
parsed_date = isoparse(
|
338
|
-
retry_after_header
|
339
|
-
).astimezone() # converts to local time
|
340
|
-
diff = (parsed_date - datetime.now().astimezone()).total_seconds()
|
341
|
-
if diff > 0:
|
342
|
-
return min(diff, self._max_backoff_wait)
|
343
|
-
except ValueError:
|
344
|
-
pass
|
345
|
-
|
346
|
-
backoff = self._base_delay * (2 ** (attempts_made - 1))
|
347
|
-
jitter = (backoff * self._jitter_ratio) * random.choice([1, -1])
|
404
|
+
# Check custom retry headers first, then fall back to Retry-After
|
405
|
+
if self._retry_config.respect_retry_after_header:
|
406
|
+
for header_name in self._retry_config.retry_after_headers:
|
407
|
+
if header_value := (headers.get(header_name) or "").strip():
|
408
|
+
sleep_time = self._parse_retry_header(header_value)
|
409
|
+
if sleep_time is not None:
|
410
|
+
return min(sleep_time, self._retry_config.max_backoff_wait)
|
411
|
+
|
412
|
+
# Fall back to exponential backoff
|
413
|
+
backoff = self._retry_config.base_delay * (2 ** (attempts_made - 1))
|
414
|
+
jitter = (backoff * self._retry_config.jitter_ratio) * random.choice([1, -1])
|
348
415
|
total_backoff = backoff + jitter
|
349
|
-
return min(total_backoff, self.
|
416
|
+
return min(total_backoff, self._retry_config.max_backoff_wait)
|
417
|
+
|
418
|
+
def _parse_retry_header(self, header_value: str) -> Optional[float]:
|
419
|
+
"""Parse retry header value and return sleep time in seconds.
|
420
|
+
|
421
|
+
Args:
|
422
|
+
header_value: The header value to parse (e.g., "30", "2023-12-01T12:00:00Z")
|
423
|
+
|
424
|
+
Returns:
|
425
|
+
Sleep time in seconds if parsing succeeds, None if the header value cannot be parsed
|
426
|
+
"""
|
427
|
+
if header_value.isdigit():
|
428
|
+
return float(header_value)
|
429
|
+
|
430
|
+
try:
|
431
|
+
# Try to parse as ISO date (common for rate limit headers like X-RateLimit-Reset)
|
432
|
+
parsed_date = isoparse(header_value).astimezone()
|
433
|
+
diff = (parsed_date - datetime.now().astimezone()).total_seconds()
|
434
|
+
if diff > 0:
|
435
|
+
return diff
|
436
|
+
except ValueError:
|
437
|
+
pass
|
438
|
+
|
439
|
+
return None
|
350
440
|
|
351
441
|
async def _retry_operation_async(
|
352
442
|
self,
|
353
443
|
request: httpx.Request,
|
354
444
|
send_method: Callable[..., Coroutine[Any, Any, httpx.Response]],
|
355
445
|
) -> httpx.Response:
|
356
|
-
remaining_attempts = self.
|
446
|
+
remaining_attempts = self._retry_config.max_attempts
|
357
447
|
attempts_made = 0
|
358
448
|
response: httpx.Response | None = None
|
359
449
|
error: Exception | None = None
|
@@ -403,7 +493,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|
403
493
|
request: httpx.Request,
|
404
494
|
send_method: Callable[..., httpx.Response],
|
405
495
|
) -> httpx.Response:
|
406
|
-
remaining_attempts = self.
|
496
|
+
remaining_attempts = self._retry_config.max_attempts
|
407
497
|
attempts_made = 0
|
408
498
|
response: httpx.Response | None = None
|
409
499
|
error: Exception | None = None
|
@@ -0,0 +1,309 @@
|
|
1
|
+
import pytest
|
2
|
+
from unittest.mock import Mock
|
3
|
+
from http import HTTPStatus
|
4
|
+
import httpx
|
5
|
+
|
6
|
+
from port_ocean.helpers.retry import (
|
7
|
+
RetryConfig,
|
8
|
+
RetryTransport,
|
9
|
+
register_retry_config_callback,
|
10
|
+
register_on_retry_callback,
|
11
|
+
)
|
12
|
+
import port_ocean.helpers.retry as retry_module
|
13
|
+
|
14
|
+
|
15
|
+
class TestRetryConfig:
|
16
|
+
def test_default_configuration(self) -> None:
|
17
|
+
"""Test RetryConfig with default parameters."""
|
18
|
+
config = RetryConfig()
|
19
|
+
|
20
|
+
assert config.max_attempts == 10
|
21
|
+
assert config.max_backoff_wait == 60.0
|
22
|
+
assert config.base_delay == 0.1
|
23
|
+
assert config.jitter_ratio == 0.1
|
24
|
+
assert config.respect_retry_after_header is True
|
25
|
+
assert config.retryable_methods == frozenset(
|
26
|
+
["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
|
27
|
+
)
|
28
|
+
assert config.retry_status_codes == frozenset(
|
29
|
+
[
|
30
|
+
HTTPStatus.TOO_MANY_REQUESTS,
|
31
|
+
HTTPStatus.BAD_GATEWAY,
|
32
|
+
HTTPStatus.SERVICE_UNAVAILABLE,
|
33
|
+
HTTPStatus.GATEWAY_TIMEOUT,
|
34
|
+
HTTPStatus.UNAUTHORIZED,
|
35
|
+
HTTPStatus.BAD_REQUEST,
|
36
|
+
]
|
37
|
+
)
|
38
|
+
assert config.retry_after_headers == ["Retry-After"]
|
39
|
+
|
40
|
+
def test_custom_configuration(self) -> None:
|
41
|
+
"""Test RetryConfig with custom parameters."""
|
42
|
+
config = RetryConfig(
|
43
|
+
max_attempts=5,
|
44
|
+
max_backoff_wait=30.0,
|
45
|
+
base_delay=2.0,
|
46
|
+
jitter_ratio=0.2,
|
47
|
+
respect_retry_after_header=False,
|
48
|
+
retryable_methods=["GET", "POST"],
|
49
|
+
retry_after_headers=["X-Custom-Retry", "Retry-After"],
|
50
|
+
additional_retry_status_codes=[418, 420],
|
51
|
+
)
|
52
|
+
|
53
|
+
assert config.max_attempts == 5
|
54
|
+
assert config.max_backoff_wait == 30.0
|
55
|
+
assert config.base_delay == 2.0
|
56
|
+
assert config.jitter_ratio == 0.2
|
57
|
+
assert config.respect_retry_after_header is False
|
58
|
+
assert config.retryable_methods == frozenset(["GET", "POST"])
|
59
|
+
# Should include defaults + additional codes
|
60
|
+
expected_codes = frozenset(
|
61
|
+
[
|
62
|
+
HTTPStatus.TOO_MANY_REQUESTS,
|
63
|
+
HTTPStatus.BAD_GATEWAY,
|
64
|
+
HTTPStatus.SERVICE_UNAVAILABLE,
|
65
|
+
HTTPStatus.GATEWAY_TIMEOUT,
|
66
|
+
HTTPStatus.UNAUTHORIZED,
|
67
|
+
HTTPStatus.BAD_REQUEST,
|
68
|
+
418,
|
69
|
+
420,
|
70
|
+
]
|
71
|
+
)
|
72
|
+
assert config.retry_status_codes == expected_codes
|
73
|
+
assert config.retry_after_headers == ["X-Custom-Retry", "Retry-After"]
|
74
|
+
|
75
|
+
def test_additional_status_codes(self) -> None:
|
76
|
+
"""Test that additional status codes extend defaults."""
|
77
|
+
config = RetryConfig(
|
78
|
+
additional_retry_status_codes=[418, 420],
|
79
|
+
)
|
80
|
+
|
81
|
+
# Should include defaults + additional codes
|
82
|
+
expected_codes = frozenset(
|
83
|
+
[
|
84
|
+
HTTPStatus.TOO_MANY_REQUESTS,
|
85
|
+
HTTPStatus.BAD_GATEWAY,
|
86
|
+
HTTPStatus.SERVICE_UNAVAILABLE,
|
87
|
+
HTTPStatus.GATEWAY_TIMEOUT,
|
88
|
+
HTTPStatus.UNAUTHORIZED,
|
89
|
+
HTTPStatus.BAD_REQUEST,
|
90
|
+
418,
|
91
|
+
420,
|
92
|
+
]
|
93
|
+
)
|
94
|
+
assert config.retry_status_codes == expected_codes
|
95
|
+
|
96
|
+
def test_invalid_jitter_ratio(self) -> None:
|
97
|
+
"""Test that invalid jitter ratio raises ValueError."""
|
98
|
+
with pytest.raises(
|
99
|
+
ValueError, match="Jitter ratio should be between 0 and 0.5"
|
100
|
+
):
|
101
|
+
RetryConfig(jitter_ratio=0.6)
|
102
|
+
|
103
|
+
with pytest.raises(
|
104
|
+
ValueError, match="Jitter ratio should be between 0 and 0.5"
|
105
|
+
):
|
106
|
+
RetryConfig(jitter_ratio=-0.1)
|
107
|
+
|
108
|
+
def test_empty_additional_status_codes(self) -> None:
|
109
|
+
"""Test that empty additional status codes work correctly."""
|
110
|
+
config = RetryConfig(additional_retry_status_codes=[])
|
111
|
+
assert config.retry_status_codes == frozenset(
|
112
|
+
[
|
113
|
+
HTTPStatus.TOO_MANY_REQUESTS,
|
114
|
+
HTTPStatus.BAD_GATEWAY,
|
115
|
+
HTTPStatus.SERVICE_UNAVAILABLE,
|
116
|
+
HTTPStatus.GATEWAY_TIMEOUT,
|
117
|
+
HTTPStatus.UNAUTHORIZED,
|
118
|
+
HTTPStatus.BAD_REQUEST,
|
119
|
+
]
|
120
|
+
)
|
121
|
+
|
122
|
+
|
123
|
+
class TestRetryConfigCallback:
|
124
|
+
def setup_method(self) -> None:
|
125
|
+
"""Reset global callback state before each test."""
|
126
|
+
retry_module._RETRY_CONFIG_CALLBACK = None
|
127
|
+
retry_module._ON_RETRY_CALLBACK = None
|
128
|
+
|
129
|
+
def test_register_retry_config_callback(self) -> None:
|
130
|
+
"""Test registering a retry config callback."""
|
131
|
+
|
132
|
+
def mock_callback() -> RetryConfig:
|
133
|
+
return RetryConfig(max_attempts=5)
|
134
|
+
|
135
|
+
register_retry_config_callback(mock_callback)
|
136
|
+
|
137
|
+
assert retry_module._RETRY_CONFIG_CALLBACK is mock_callback
|
138
|
+
config = retry_module._RETRY_CONFIG_CALLBACK()
|
139
|
+
assert config.max_attempts == 5
|
140
|
+
|
141
|
+
def test_register_on_retry_callback(self) -> None:
|
142
|
+
"""Test registering an on retry callback."""
|
143
|
+
|
144
|
+
def mock_callback(request: httpx.Request) -> httpx.Request:
|
145
|
+
return request
|
146
|
+
|
147
|
+
register_on_retry_callback(mock_callback)
|
148
|
+
|
149
|
+
assert retry_module._ON_RETRY_CALLBACK is mock_callback
|
150
|
+
|
151
|
+
def test_callback_overwrite(self) -> None:
|
152
|
+
"""Test that registering a new callback overwrites the previous one."""
|
153
|
+
|
154
|
+
def callback1() -> RetryConfig:
|
155
|
+
return RetryConfig(max_attempts=1)
|
156
|
+
|
157
|
+
def callback2() -> RetryConfig:
|
158
|
+
return RetryConfig(max_attempts=2)
|
159
|
+
|
160
|
+
register_retry_config_callback(callback1)
|
161
|
+
assert retry_module._RETRY_CONFIG_CALLBACK is not None
|
162
|
+
assert retry_module._RETRY_CONFIG_CALLBACK().max_attempts == 1
|
163
|
+
|
164
|
+
register_retry_config_callback(callback2)
|
165
|
+
assert retry_module._RETRY_CONFIG_CALLBACK is not None
|
166
|
+
assert retry_module._RETRY_CONFIG_CALLBACK().max_attempts == 2
|
167
|
+
|
168
|
+
|
169
|
+
class TestRetryTransport:
|
170
|
+
def setup_method(self) -> None:
|
171
|
+
"""Reset global callback state before each test."""
|
172
|
+
retry_module._RETRY_CONFIG_CALLBACK = None
|
173
|
+
retry_module._ON_RETRY_CALLBACK = None
|
174
|
+
|
175
|
+
def test_retry_transport_with_direct_config(self) -> None:
|
176
|
+
"""Test RetryTransport with direct retry_config parameter."""
|
177
|
+
mock_transport = Mock()
|
178
|
+
config = RetryConfig(max_attempts=5)
|
179
|
+
|
180
|
+
transport = RetryTransport(
|
181
|
+
wrapped_transport=mock_transport,
|
182
|
+
retry_config=config,
|
183
|
+
)
|
184
|
+
|
185
|
+
assert transport._retry_config is config
|
186
|
+
assert transport._retry_config.max_attempts == 5
|
187
|
+
|
188
|
+
def test_retry_transport_with_callback(self) -> None:
|
189
|
+
"""Test RetryTransport using registered callback."""
|
190
|
+
|
191
|
+
def mock_callback() -> RetryConfig:
|
192
|
+
return RetryConfig(max_attempts=7)
|
193
|
+
|
194
|
+
register_retry_config_callback(mock_callback)
|
195
|
+
|
196
|
+
mock_transport = Mock()
|
197
|
+
transport = RetryTransport(wrapped_transport=mock_transport)
|
198
|
+
|
199
|
+
assert transport._retry_config.max_attempts == 7
|
200
|
+
|
201
|
+
def test_retry_transport_priority_order(self) -> None:
|
202
|
+
"""Test that direct config takes priority over callback."""
|
203
|
+
|
204
|
+
def mock_callback() -> RetryConfig:
|
205
|
+
return RetryConfig(max_attempts=7)
|
206
|
+
|
207
|
+
register_retry_config_callback(mock_callback)
|
208
|
+
|
209
|
+
mock_transport = Mock()
|
210
|
+
direct_config = RetryConfig(max_attempts=5)
|
211
|
+
|
212
|
+
transport = RetryTransport(
|
213
|
+
wrapped_transport=mock_transport,
|
214
|
+
retry_config=direct_config,
|
215
|
+
)
|
216
|
+
|
217
|
+
assert transport._retry_config.max_attempts == 5 # Direct config wins
|
218
|
+
|
219
|
+
def test_retry_transport_default_config(self) -> None:
|
220
|
+
"""Test RetryTransport with no config or callback."""
|
221
|
+
mock_transport = Mock()
|
222
|
+
transport = RetryTransport(wrapped_transport=mock_transport)
|
223
|
+
|
224
|
+
assert transport._retry_config.max_attempts == 10 # Default value
|
225
|
+
assert transport._retry_config.retry_after_headers == ["Retry-After"]
|
226
|
+
|
227
|
+
def test_is_retryable_method(self) -> None:
|
228
|
+
"""Test _is_retryable_method functionality."""
|
229
|
+
mock_transport = Mock()
|
230
|
+
transport = RetryTransport(wrapped_transport=mock_transport)
|
231
|
+
|
232
|
+
# Test with default retryable methods
|
233
|
+
mock_request = Mock()
|
234
|
+
mock_request.method = "GET"
|
235
|
+
mock_request.extensions = {}
|
236
|
+
|
237
|
+
assert transport._is_retryable_method(mock_request) is True
|
238
|
+
|
239
|
+
mock_request.method = "POST"
|
240
|
+
assert transport._is_retryable_method(mock_request) is False
|
241
|
+
|
242
|
+
# Test with retryable extension
|
243
|
+
mock_request.extensions = {"retryable": True}
|
244
|
+
assert transport._is_retryable_method(mock_request) is True
|
245
|
+
|
246
|
+
def test_should_retry(self) -> None:
|
247
|
+
"""Test _should_retry functionality."""
|
248
|
+
mock_transport = Mock()
|
249
|
+
transport = RetryTransport(wrapped_transport=mock_transport)
|
250
|
+
|
251
|
+
mock_response = Mock()
|
252
|
+
mock_response.status_code = 429
|
253
|
+
assert transport._should_retry(mock_response) is True
|
254
|
+
|
255
|
+
mock_response.status_code = 200
|
256
|
+
assert transport._should_retry(mock_response) is False
|
257
|
+
|
258
|
+
def test_parse_retry_header(self) -> None:
|
259
|
+
"""Test _parse_retry_header functionality."""
|
260
|
+
mock_transport = Mock()
|
261
|
+
transport = RetryTransport(wrapped_transport=mock_transport)
|
262
|
+
|
263
|
+
# Test numeric seconds
|
264
|
+
assert transport._parse_retry_header("30") == 30.0
|
265
|
+
|
266
|
+
# Test invalid numeric
|
267
|
+
assert transport._parse_retry_header("invalid") is None
|
268
|
+
|
269
|
+
# Test ISO date (would need proper date parsing test)
|
270
|
+
# This is a basic test - actual date parsing would need more complex setup
|
271
|
+
assert (
|
272
|
+
transport._parse_retry_header("2023-12-01T12:00:00Z") is None
|
273
|
+
) # Will fail parsing
|
274
|
+
|
275
|
+
|
276
|
+
class TestRetryConfigIntegration:
|
277
|
+
"""Integration tests for retry configuration."""
|
278
|
+
|
279
|
+
def setup_method(self) -> None:
|
280
|
+
"""Reset global callback state before each test."""
|
281
|
+
retry_module._RETRY_CONFIG_CALLBACK = None
|
282
|
+
retry_module._ON_RETRY_CALLBACK = None
|
283
|
+
|
284
|
+
def test_integration_style_config(self) -> None:
|
285
|
+
"""Test configuration similar to GitHub integration."""
|
286
|
+
|
287
|
+
def integration_retry_config() -> RetryConfig:
|
288
|
+
return RetryConfig(
|
289
|
+
max_attempts=10,
|
290
|
+
max_backoff_wait=60.0,
|
291
|
+
base_delay=1.0,
|
292
|
+
jitter_ratio=0.1,
|
293
|
+
respect_retry_after_header=True,
|
294
|
+
retry_after_headers=["X-RateLimit-Reset", "Retry-After"],
|
295
|
+
additional_retry_status_codes=[HTTPStatus.FORBIDDEN],
|
296
|
+
)
|
297
|
+
|
298
|
+
register_retry_config_callback(integration_retry_config)
|
299
|
+
|
300
|
+
mock_transport = Mock()
|
301
|
+
transport = RetryTransport(wrapped_transport=mock_transport)
|
302
|
+
|
303
|
+
assert transport._retry_config.max_attempts == 10
|
304
|
+
assert transport._retry_config.base_delay == 1.0
|
305
|
+
assert transport._retry_config.retry_after_headers == [
|
306
|
+
"X-RateLimit-Reset",
|
307
|
+
"Retry-After",
|
308
|
+
]
|
309
|
+
assert HTTPStatus.FORBIDDEN in transport._retry_config.retry_status_codes
|
@@ -123,7 +123,7 @@ port_ocean/core/integrations/mixins/events.py,sha256=2L7P3Jhp8XBqddh2_o9Cn4N261n
|
|
123
123
|
port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
|
124
124
|
port_ocean/core/integrations/mixins/live_events.py,sha256=zM24dhNc7uHx9XYZ6toVhDADPA90EnpOmZxgDegFZbA,4196
|
125
125
|
port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
|
126
|
-
port_ocean/core/integrations/mixins/sync_raw.py,sha256=
|
126
|
+
port_ocean/core/integrations/mixins/sync_raw.py,sha256=EJLF0vMFVkQi4zJPQAFtg-VkQJsOGbya3yhEfWu2L1c,40734
|
127
127
|
port_ocean/core/integrations/mixins/utils.py,sha256=ytnFX7Lyv6N3CgBnOXxYaI1cRDq5Z4NDrVFiwE6bn-M,5250
|
128
128
|
port_ocean/core/models.py,sha256=DNbKpStMINI2lIekKprTqBevqkw_wFuFayN19w1aDfQ,2893
|
129
129
|
port_ocean/core/ocean_types.py,sha256=bkLlTd8XfJK6_JDl0eXUHfE_NygqgiInSMwJ4YJH01Q,1399
|
@@ -140,10 +140,10 @@ port_ocean/exceptions/port_defaults.py,sha256=2a7Koy541KxMan33mU-gbauUxsumG3NT4i
|
|
140
140
|
port_ocean/exceptions/utils.py,sha256=gjOqpi-HpY1l4WlMFsGA9yzhxDhajhoGGdDDyGbLnqI,197
|
141
141
|
port_ocean/exceptions/webhook_processor.py,sha256=4SnkVzVwiacH_Ip4qs1hRHa6GanhnojW_TLTdQQtm7Y,363
|
142
142
|
port_ocean/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
143
|
-
port_ocean/helpers/async_client.py,sha256=
|
143
|
+
port_ocean/helpers/async_client.py,sha256=M8gKUjX8ZwRbmJ-U6KNq-p-nfGr0CwHdS0eN_pbZAJ0,2103
|
144
144
|
port_ocean/helpers/metric/metric.py,sha256=-dw7-Eqr65AZwv0M-xPaAk98g_JS16ICBc9_UkycFbE,14543
|
145
145
|
port_ocean/helpers/metric/utils.py,sha256=1lAgrxnZLuR_wUNDyPOPzLrm32b8cDdioob2lvnPQ1A,1619
|
146
|
-
port_ocean/helpers/retry.py,sha256=
|
146
|
+
port_ocean/helpers/retry.py,sha256=QM04mzaevIUlg8HnHjeY9UT_D4k26BHx3hVkCjV_jnY,21675
|
147
147
|
port_ocean/helpers/stream.py,sha256=_UwsThzXynxWzL8OlBT1pmb2evZBi9HaaqeAGNuTuOI,2338
|
148
148
|
port_ocean/log/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
149
149
|
port_ocean/log/handlers.py,sha256=LJ1WAfq7wYCrBpeLPihMKmWjdSahKKXNHFMRYkbk0Co,3630
|
@@ -190,6 +190,7 @@ port_ocean/tests/helpers/integration.py,sha256=_RxS-RHpu11lrbhUXYPZp862HLWx8AoD7
|
|
190
190
|
port_ocean/tests/helpers/ocean_app.py,sha256=N06vcNI1klqdcNFq-PXL5vm77u-hODsOSXnj9p8d1AI,2249
|
191
191
|
port_ocean/tests/helpers/port_client.py,sha256=S0CXvZWUoHFWWQUOEgdkDammK9Fs3R06wx0flaMrTsg,647
|
192
192
|
port_ocean/tests/helpers/smoke_test.py,sha256=_9aJJFRfuGJEg2D2YQJVJRmpreS6gEPHHQq8Q01x4aQ,2697
|
193
|
+
port_ocean/tests/helpers/test_retry.py,sha256=c4kS5XrQNv8r-uqPLbWVGvudTy3b_dHxhcCjm7ZWeAQ,11033
|
193
194
|
port_ocean/tests/log/test_handlers.py,sha256=x2P2Hd6Cb3sQafIE3TRGltbbHeiFHaiEjwRn9py_03g,2165
|
194
195
|
port_ocean/tests/test_metric.py,sha256=gDdeJcqJDQ_o3VrYrW23iZyw2NuUsyATdrygSXhcDuQ,8096
|
195
196
|
port_ocean/tests/test_ocean.py,sha256=bsXKGTVEjwLSbR7-qSmI4GZ-EzDo0eBE3TNSMsWzYxM,1502
|
@@ -207,8 +208,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
|
|
207
208
|
port_ocean/utils/signal.py,sha256=mMVq-1Ab5YpNiqN4PkiyTGlV_G0wkUDMMjTZp5z3pb0,1514
|
208
209
|
port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
|
209
210
|
port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
|
210
|
-
port_ocean-0.
|
211
|
-
port_ocean-0.
|
212
|
-
port_ocean-0.
|
213
|
-
port_ocean-0.
|
214
|
-
port_ocean-0.
|
211
|
+
port_ocean-0.28.1.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
212
|
+
port_ocean-0.28.1.dist-info/METADATA,sha256=PSPEzll-3f7zXAELk1jUxqAlzwsxtxLksle9ITyREwg,7015
|
213
|
+
port_ocean-0.28.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
214
|
+
port_ocean-0.28.1.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
|
215
|
+
port_ocean-0.28.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|