http-misc 0.0.2__tar.gz → 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {http_misc-0.0.2 → http_misc-1.0.1}/PKG-INFO +3 -1
- {http_misc-0.0.2 → http_misc-1.0.1}/http_misc/http_utils.py +9 -3
- http_misc-1.0.1/http_misc/logger.py +11 -0
- http_misc-1.0.1/http_misc/retry_policy.py +120 -0
- http_misc-1.0.1/http_misc/services.py +134 -0
- {http_misc-0.0.2 → http_misc-1.0.1}/http_misc.egg-info/PKG-INFO +3 -1
- {http_misc-0.0.2 → http_misc-1.0.1}/http_misc.egg-info/SOURCES.txt +5 -1
- {http_misc-0.0.2 → http_misc-1.0.1}/http_misc.egg-info/requires.txt +2 -0
- {http_misc-0.0.2 → http_misc-1.0.1}/setup.py +4 -2
- http_misc-1.0.1/tests/test_retry_policy.py +66 -0
- http_misc-1.0.1/tests/test_services.py +118 -0
- http_misc-0.0.2/http_misc/services.py +0 -237
- {http_misc-0.0.2 → http_misc-1.0.1}/README.md +0 -0
- {http_misc-0.0.2 → http_misc-1.0.1}/http_misc/__init__.py +0 -0
- {http_misc-0.0.2 → http_misc-1.0.1}/http_misc/errors.py +0 -0
- {http_misc-0.0.2 → http_misc-1.0.1}/http_misc.egg-info/dependency_links.txt +0 -0
- {http_misc-0.0.2 → http_misc-1.0.1}/http_misc.egg-info/top_level.txt +0 -0
- {http_misc-0.0.2 → http_misc-1.0.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: http-misc
|
|
3
|
-
Version:
|
|
3
|
+
Version: 1.0.1
|
|
4
4
|
Summary: Утилитарный пакет межсервисного взаимодействия по протоколу HTTP
|
|
5
5
|
Author: Anton Gorinenko
|
|
6
6
|
Author-email: anton.gorinenko@gmail.com
|
|
@@ -18,6 +18,8 @@ Requires-Dist: pytest; extra == "test"
|
|
|
18
18
|
Requires-Dist: python-dotenv; extra == "test"
|
|
19
19
|
Requires-Dist: envparse; extra == "test"
|
|
20
20
|
Requires-Dist: pytest-asyncio; extra == "test"
|
|
21
|
+
Requires-Dist: pytest-mock; extra == "test"
|
|
22
|
+
Requires-Dist: pytest-env; extra == "test"
|
|
21
23
|
Dynamic: author
|
|
22
24
|
Dynamic: author-email
|
|
23
25
|
Dynamic: classifier
|
|
@@ -2,8 +2,9 @@ import datetime
|
|
|
2
2
|
import decimal
|
|
3
3
|
import json
|
|
4
4
|
import uuid
|
|
5
|
+
from typing import Optional
|
|
5
6
|
|
|
6
|
-
from http_misc import services, errors
|
|
7
|
+
from http_misc import services, errors, retry_policy
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
def default_encoder(obj):
|
|
@@ -31,9 +32,14 @@ def join_str(*args, sep: str | None = '/', append_last_sep: bool | None = False)
|
|
|
31
32
|
return url
|
|
32
33
|
|
|
33
34
|
|
|
34
|
-
async def send_and_validate(service: 'services.BaseService', request, expected_status: int | None = 200
|
|
35
|
+
async def send_and_validate(service: 'services.BaseService', request, expected_status: int | None = 200,
|
|
36
|
+
policy: Optional['retry_policy.AsyncRetryPolicy'] = None):
|
|
35
37
|
""" Вызов внешнего сервиса и проверка его статуса"""
|
|
36
|
-
|
|
38
|
+
if policy:
|
|
39
|
+
response = await policy.apply(service.send_request, **request)
|
|
40
|
+
else:
|
|
41
|
+
response = await service.send_request(**request)
|
|
42
|
+
|
|
37
43
|
if response.status != expected_status:
|
|
38
44
|
raise errors.InteractionError('Произошла ошибка при вызове внешнего сервиса',
|
|
39
45
|
status_code=response.status, response=response.response_data)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import random
|
|
3
|
+
import uuid
|
|
4
|
+
from abc import ABC
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from time import sleep
|
|
7
|
+
|
|
8
|
+
from http_misc.errors import RetryError, MaxRetryError
|
|
9
|
+
from http_misc.logger import get_logger
|
|
10
|
+
|
|
11
|
+
logger = get_logger('retry_policy')
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class BaseRetryPolicy(ABC):
|
|
15
|
+
"""
|
|
16
|
+
Базовая политика действий
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, max_retry: int | None = 9, backoff_factor: float | None = 0.3, jitter: float | None = 0.1):
|
|
20
|
+
""" Базовая политика действий
|
|
21
|
+
:param max_retry: максимальное количество повторений(без учета основного вызова)
|
|
22
|
+
:param backoff_factor: коэффициент задержки попыток повторных вызовов
|
|
23
|
+
:param jitter: коэффициент "дрожания" повторных вызовов
|
|
24
|
+
"""
|
|
25
|
+
self.max_retry = max_retry
|
|
26
|
+
self.backoff_factor = backoff_factor
|
|
27
|
+
self.jitter = jitter
|
|
28
|
+
|
|
29
|
+
self.request_count_manager = RequestCountManager()
|
|
30
|
+
|
|
31
|
+
def _on_retry_error(self, current_step: int, request_id: uuid.UUID) -> float:
|
|
32
|
+
if current_step >= self.max_retry:
|
|
33
|
+
raise MaxRetryError(f'Exceeded the maximum number of attempts {self.max_retry}.')
|
|
34
|
+
|
|
35
|
+
sleep_seconds = self.backoff_factor * (2 ** (current_step - 1))
|
|
36
|
+
sleep_seconds += random.normalvariate(0, sleep_seconds * self.jitter)
|
|
37
|
+
self.request_count_manager.inc(request_id)
|
|
38
|
+
return sleep_seconds
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class AsyncRetryPolicy(BaseRetryPolicy):
|
|
42
|
+
"""
|
|
43
|
+
Политика повторов асинхронных действий
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
async def apply(self, action: Callable, *args, **kwargs):
|
|
47
|
+
""" Выполнение асинхронного действия """
|
|
48
|
+
request_id = self.request_count_manager.add()
|
|
49
|
+
try:
|
|
50
|
+
while True:
|
|
51
|
+
current_step = self.request_count_manager.get(request_id)
|
|
52
|
+
if current_step > 0:
|
|
53
|
+
logger.debug('Step %s. Repeat action #%s.', request_id)
|
|
54
|
+
try:
|
|
55
|
+
return await action(*args, **kwargs)
|
|
56
|
+
except RetryError:
|
|
57
|
+
sleep_seconds = self._on_retry_error(current_step, request_id)
|
|
58
|
+
await asyncio.sleep(sleep_seconds)
|
|
59
|
+
finally:
|
|
60
|
+
self.request_count_manager.pop(request_id)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class RetryPolicy(BaseRetryPolicy):
|
|
64
|
+
"""
|
|
65
|
+
Политика повторов синхронных действий
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
def apply(self, action: Callable, *args, **kwargs):
|
|
69
|
+
""" Выполнение синхронного действия """
|
|
70
|
+
request_id = self.request_count_manager.add()
|
|
71
|
+
try:
|
|
72
|
+
while True:
|
|
73
|
+
current_step = self.request_count_manager.get(request_id)
|
|
74
|
+
if current_step > 0:
|
|
75
|
+
logger.debug('Step %s. Repeat action #%s.', request_id)
|
|
76
|
+
try:
|
|
77
|
+
return action(*args, **kwargs)
|
|
78
|
+
except RetryError:
|
|
79
|
+
sleep_seconds = self._on_retry_error(current_step, request_id)
|
|
80
|
+
sleep(sleep_seconds)
|
|
81
|
+
finally:
|
|
82
|
+
self.request_count_manager.pop(request_id)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class RequestCountManager:
|
|
86
|
+
def __init__(self):
|
|
87
|
+
self._requests: dict[uuid.UUID, int] = {}
|
|
88
|
+
|
|
89
|
+
def get_requests(self):
|
|
90
|
+
return self._requests
|
|
91
|
+
|
|
92
|
+
def add(self) -> uuid.UUID:
|
|
93
|
+
""" Инициализация запроса """
|
|
94
|
+
request_id = uuid.uuid4()
|
|
95
|
+
self._requests[request_id] = 0
|
|
96
|
+
return request_id
|
|
97
|
+
|
|
98
|
+
def exist(self, request_id: uuid.UUID) -> bool:
|
|
99
|
+
""" Проверка наличия запроса """
|
|
100
|
+
if request_id not in self._requests:
|
|
101
|
+
raise KeyError(f'Request {request_id} not in registry.')
|
|
102
|
+
|
|
103
|
+
return True
|
|
104
|
+
|
|
105
|
+
def pop(self, request_id: uuid.UUID) -> int | None:
|
|
106
|
+
""" Удаление запроса """
|
|
107
|
+
self.exist(request_id)
|
|
108
|
+
return self._requests.pop(request_id)
|
|
109
|
+
|
|
110
|
+
def get(self, request_id: uuid.UUID) -> int:
|
|
111
|
+
""" Получение количества попыток запроса """
|
|
112
|
+
self.exist(request_id)
|
|
113
|
+
return self._requests[request_id]
|
|
114
|
+
|
|
115
|
+
def inc(self, request_id: uuid.UUID) -> int:
|
|
116
|
+
""" Увеличение количества попыток на 1 """
|
|
117
|
+
self.exist(request_id)
|
|
118
|
+
self._requests[request_id] += 1
|
|
119
|
+
|
|
120
|
+
return self._requests[request_id]
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
from contextlib import asynccontextmanager
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from aiohttp import ContentTypeError, ClientSession
|
|
7
|
+
|
|
8
|
+
from http_misc import http_utils, errors
|
|
9
|
+
from http_misc.logger import get_logger
|
|
10
|
+
|
|
11
|
+
DEFAULT_RETRY_ON_STATUSES = frozenset([413, 429, 503, 504])
|
|
12
|
+
logger = get_logger('services')
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class ServiceResponse:
|
|
17
|
+
""" Ответ сервиса """
|
|
18
|
+
status: int
|
|
19
|
+
response_data: any = None
|
|
20
|
+
raw_response: any = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Transformer(ABC):
|
|
24
|
+
@abstractmethod
|
|
25
|
+
async def modify(self, *args, **kwargs):
|
|
26
|
+
""" Изменение параметров запроса или ответа """
|
|
27
|
+
return args, kwargs
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class BaseService(ABC):
|
|
31
|
+
"""
|
|
32
|
+
Abstract service
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(self, retry_on_statuses: set[int] | None = DEFAULT_RETRY_ON_STATUSES,
|
|
36
|
+
request_preproc: list[Transformer] | None = None,
|
|
37
|
+
response_preproc: list[Transformer] | None = None):
|
|
38
|
+
""" Сервис """
|
|
39
|
+
self.retry_on_statuses = retry_on_statuses
|
|
40
|
+
self.request_preproc = request_preproc
|
|
41
|
+
self.response_preproc = response_preproc
|
|
42
|
+
|
|
43
|
+
@abstractmethod
|
|
44
|
+
async def _send(self, *args, **kwargs) -> ServiceResponse:
|
|
45
|
+
"""
|
|
46
|
+
Abstract _send
|
|
47
|
+
"""
|
|
48
|
+
raise NotImplementedError('Not implemented _send method')
|
|
49
|
+
|
|
50
|
+
async def send_request(self, *args, **kwargs) -> ServiceResponse:
|
|
51
|
+
"""
|
|
52
|
+
Вызов внешнего сервиса
|
|
53
|
+
"""
|
|
54
|
+
try:
|
|
55
|
+
args, kwargs = await self._before_send(*args, **kwargs)
|
|
56
|
+
logger.debug('Send request %s; %s', args, kwargs)
|
|
57
|
+
service_response = await self._send(*args, **kwargs)
|
|
58
|
+
service_response = await self._transform_response(service_response)
|
|
59
|
+
logger.debug('Response: %s, %s', service_response.status, service_response.response_data)
|
|
60
|
+
|
|
61
|
+
if self.retry_on_statuses and service_response.status in self.retry_on_statuses:
|
|
62
|
+
raise errors.RetryError()
|
|
63
|
+
|
|
64
|
+
return service_response
|
|
65
|
+
except Exception as ex:
|
|
66
|
+
if isinstance(ex, errors.RetryError):
|
|
67
|
+
raise ex
|
|
68
|
+
else:
|
|
69
|
+
return await self._on_error(ex, *args, **kwargs)
|
|
70
|
+
|
|
71
|
+
async def _transform_response(self, response: ServiceResponse) -> ServiceResponse:
|
|
72
|
+
""" Преобразование ответа для возврата пользователю """
|
|
73
|
+
if self.response_preproc:
|
|
74
|
+
for response_preproc in self.response_preproc:
|
|
75
|
+
response = await response_preproc.modify(response)
|
|
76
|
+
return response
|
|
77
|
+
|
|
78
|
+
async def _before_send(self, *args, **kwargs):
|
|
79
|
+
""" Действие перед вызовом """
|
|
80
|
+
if self.request_preproc:
|
|
81
|
+
for request_preproc in self.request_preproc:
|
|
82
|
+
args, kwargs = await request_preproc.modify(*args, **kwargs)
|
|
83
|
+
return args, kwargs
|
|
84
|
+
|
|
85
|
+
async def _on_error(self, ex: Exception, *args, **kwargs) -> ServiceResponse:
|
|
86
|
+
"""
|
|
87
|
+
Действие на возникновение ошибки.
|
|
88
|
+
"""
|
|
89
|
+
logger.exception(ex)
|
|
90
|
+
raise ex
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class HttpService(BaseService):
|
|
94
|
+
"""
|
|
95
|
+
Вызов сервиса по протоколу http
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
def __init__(self, *args, client_session: ClientSession | None = None, **kwargs):
|
|
99
|
+
super().__init__(*args, **kwargs)
|
|
100
|
+
self.client_session = client_session
|
|
101
|
+
|
|
102
|
+
async def _send(self, request_id: uuid.UUID, *args, **kwargs) -> ServiceResponse:
|
|
103
|
+
method = kwargs.get('method', 'get')
|
|
104
|
+
url = kwargs.get('url', None)
|
|
105
|
+
if url is None:
|
|
106
|
+
raise ValueError('Url is none')
|
|
107
|
+
url = str(url)
|
|
108
|
+
|
|
109
|
+
cfg = kwargs.get('cfg', {})
|
|
110
|
+
if not isinstance(cfg, dict):
|
|
111
|
+
raise ValueError('Invalid cfg type. Must be dict.')
|
|
112
|
+
|
|
113
|
+
if url.lower().startswith('https://') and 'ssl' not in cfg:
|
|
114
|
+
cfg['ssl'] = False
|
|
115
|
+
|
|
116
|
+
async with self._use_client_session() as session:
|
|
117
|
+
async with session.request(method, url, **cfg) as response:
|
|
118
|
+
response_data = await _get_response_content(response)
|
|
119
|
+
return ServiceResponse(status=response.status, response_data=response_data, raw_response=response)
|
|
120
|
+
|
|
121
|
+
@asynccontextmanager
|
|
122
|
+
async def _use_client_session(self):
|
|
123
|
+
if self.client_session is not None:
|
|
124
|
+
yield self.client_session
|
|
125
|
+
else:
|
|
126
|
+
async with ClientSession(json_serialize=http_utils.json_dumps) as session:
|
|
127
|
+
yield session
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
async def _get_response_content(response):
|
|
131
|
+
try:
|
|
132
|
+
return await response.json()
|
|
133
|
+
except ContentTypeError:
|
|
134
|
+
return await response.text()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: http-misc
|
|
3
|
-
Version:
|
|
3
|
+
Version: 1.0.1
|
|
4
4
|
Summary: Утилитарный пакет межсервисного взаимодействия по протоколу HTTP
|
|
5
5
|
Author: Anton Gorinenko
|
|
6
6
|
Author-email: anton.gorinenko@gmail.com
|
|
@@ -18,6 +18,8 @@ Requires-Dist: pytest; extra == "test"
|
|
|
18
18
|
Requires-Dist: python-dotenv; extra == "test"
|
|
19
19
|
Requires-Dist: envparse; extra == "test"
|
|
20
20
|
Requires-Dist: pytest-asyncio; extra == "test"
|
|
21
|
+
Requires-Dist: pytest-mock; extra == "test"
|
|
22
|
+
Requires-Dist: pytest-env; extra == "test"
|
|
21
23
|
Dynamic: author
|
|
22
24
|
Dynamic: author-email
|
|
23
25
|
Dynamic: classifier
|
|
@@ -3,9 +3,13 @@ setup.py
|
|
|
3
3
|
http_misc/__init__.py
|
|
4
4
|
http_misc/errors.py
|
|
5
5
|
http_misc/http_utils.py
|
|
6
|
+
http_misc/logger.py
|
|
7
|
+
http_misc/retry_policy.py
|
|
6
8
|
http_misc/services.py
|
|
7
9
|
http_misc.egg-info/PKG-INFO
|
|
8
10
|
http_misc.egg-info/SOURCES.txt
|
|
9
11
|
http_misc.egg-info/dependency_links.txt
|
|
10
12
|
http_misc.egg-info/requires.txt
|
|
11
|
-
http_misc.egg-info/top_level.txt
|
|
13
|
+
http_misc.egg-info/top_level.txt
|
|
14
|
+
tests/test_retry_policy.py
|
|
15
|
+
tests/test_services.py
|
|
@@ -4,7 +4,7 @@ import setuptools
|
|
|
4
4
|
|
|
5
5
|
setuptools.setup(
|
|
6
6
|
name='http-misc',
|
|
7
|
-
version='
|
|
7
|
+
version='1.0.1',
|
|
8
8
|
author='Anton Gorinenko',
|
|
9
9
|
author_email='anton.gorinenko@gmail.com',
|
|
10
10
|
description='Утилитарный пакет межсервисного взаимодействия по протоколу HTTP',
|
|
@@ -28,7 +28,9 @@ setuptools.setup(
|
|
|
28
28
|
'pytest',
|
|
29
29
|
'python-dotenv',
|
|
30
30
|
'envparse',
|
|
31
|
-
'pytest-asyncio'
|
|
31
|
+
'pytest-asyncio',
|
|
32
|
+
'pytest-mock',
|
|
33
|
+
'pytest-env'
|
|
32
34
|
]
|
|
33
35
|
},
|
|
34
36
|
python_requires='>=3.10',
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
from unittest.mock import MagicMock, AsyncMock
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from http_misc.errors import RetryError, MaxRetryError
|
|
6
|
+
from http_misc.retry_policy import RetryPolicy, AsyncRetryPolicy
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@pytest.mark.parametrize('clazz', [RetryPolicy, AsyncRetryPolicy])
|
|
10
|
+
async def test_async_apply(clazz):
|
|
11
|
+
""" Выполнение асинхронного действия. Успех """
|
|
12
|
+
policy = clazz()
|
|
13
|
+
is_async_policy = isinstance(policy, AsyncRetryPolicy)
|
|
14
|
+
|
|
15
|
+
some_action = AsyncMock() if is_async_policy else MagicMock()
|
|
16
|
+
some_action.return_value = '123'
|
|
17
|
+
|
|
18
|
+
if is_async_policy:
|
|
19
|
+
result = await policy.apply(some_action)
|
|
20
|
+
else:
|
|
21
|
+
result = policy.apply(some_action)
|
|
22
|
+
|
|
23
|
+
requests = policy.request_count_manager.get_requests()
|
|
24
|
+
assert result == '123'
|
|
25
|
+
assert some_action.call_count == 1
|
|
26
|
+
assert len(requests.keys()) == 0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@pytest.mark.parametrize('clazz', [RetryPolicy, AsyncRetryPolicy])
|
|
30
|
+
async def test_async_apply__retry_error(clazz):
|
|
31
|
+
""" Выполнение асинхронного действия. RetryError """
|
|
32
|
+
max_retry = 5
|
|
33
|
+
policy = clazz(max_retry=max_retry, backoff_factor=0.001, jitter=0.001)
|
|
34
|
+
is_async_policy = isinstance(policy, AsyncRetryPolicy)
|
|
35
|
+
|
|
36
|
+
some_action = AsyncMock() if is_async_policy else MagicMock()
|
|
37
|
+
some_action.side_effect = RetryError()
|
|
38
|
+
|
|
39
|
+
with pytest.raises(MaxRetryError, match=f'Exceeded the maximum number of attempts {max_retry}.'):
|
|
40
|
+
if is_async_policy:
|
|
41
|
+
await policy.apply(some_action)
|
|
42
|
+
else:
|
|
43
|
+
policy.apply(some_action)
|
|
44
|
+
|
|
45
|
+
requests = policy.request_count_manager.get_requests()
|
|
46
|
+
assert some_action.call_count == max_retry + 1
|
|
47
|
+
assert len(requests.keys()) == 0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@pytest.mark.parametrize('clazz', [RetryPolicy, AsyncRetryPolicy])
|
|
51
|
+
async def test_async_apply__error(clazz):
|
|
52
|
+
""" Выполнение асинхронного действия. Exception """
|
|
53
|
+
policy = clazz()
|
|
54
|
+
is_async_policy = isinstance(policy, AsyncRetryPolicy)
|
|
55
|
+
|
|
56
|
+
some_action = AsyncMock() if is_async_policy else MagicMock()
|
|
57
|
+
some_action.side_effect = Exception('Test')
|
|
58
|
+
with pytest.raises(Exception, match=f'Test'):
|
|
59
|
+
if is_async_policy:
|
|
60
|
+
await policy.apply(some_action)
|
|
61
|
+
else:
|
|
62
|
+
policy.apply(some_action)
|
|
63
|
+
|
|
64
|
+
requests = policy.request_count_manager.get_requests()
|
|
65
|
+
assert some_action.call_count == 1
|
|
66
|
+
assert len(requests.keys()) == 0
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
from unittest.mock import call
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from http_misc import http_utils
|
|
5
|
+
|
|
6
|
+
from http_misc.errors import RetryError, MaxRetryError
|
|
7
|
+
from http_misc.retry_policy import AsyncRetryPolicy
|
|
8
|
+
from http_misc.services import HttpService, ServiceResponse
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
async def test_http_service(mocker):
|
|
12
|
+
response_data = {
|
|
13
|
+
'meta': {
|
|
14
|
+
'count': 5
|
|
15
|
+
},
|
|
16
|
+
'list': [
|
|
17
|
+
1, 2, 3, 4, 5
|
|
18
|
+
]
|
|
19
|
+
}
|
|
20
|
+
send_mocker = mocker.patch('http_misc.services.HttpService._send')
|
|
21
|
+
send_mocker.return_value = ServiceResponse(status=200, response_data=response_data, raw_response=None)
|
|
22
|
+
|
|
23
|
+
policy = AsyncRetryPolicy()
|
|
24
|
+
service = HttpService()
|
|
25
|
+
request = {
|
|
26
|
+
'method': 'GET',
|
|
27
|
+
'url': 'https://localhost:8000',
|
|
28
|
+
'cfg': {
|
|
29
|
+
'params': {
|
|
30
|
+
'q1': 1,
|
|
31
|
+
'q2': '2'
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
result = await policy.apply(service.send_request, **request)
|
|
36
|
+
assert result.status == 200
|
|
37
|
+
assert result.response_data == response_data
|
|
38
|
+
|
|
39
|
+
assert send_mocker.call_args_list == [
|
|
40
|
+
call(method='GET', url='https://localhost:8000', cfg={'params': {'q1': 1, 'q2': '2'}})
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
async def test_http_service__500(mocker):
|
|
45
|
+
response_data = {
|
|
46
|
+
'error': 'Error1'
|
|
47
|
+
}
|
|
48
|
+
send_mocker = mocker.patch('http_misc.services.HttpService._send')
|
|
49
|
+
send_mocker.return_value = ServiceResponse(status=500, response_data=response_data, raw_response=None)
|
|
50
|
+
|
|
51
|
+
policy = AsyncRetryPolicy()
|
|
52
|
+
service = HttpService()
|
|
53
|
+
request = {
|
|
54
|
+
'method': 'GET',
|
|
55
|
+
'url': 'https://localhost:8000',
|
|
56
|
+
'cfg': {
|
|
57
|
+
'params': {
|
|
58
|
+
'q1': 1,
|
|
59
|
+
'q2': '2'
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
result = await http_utils.send_and_validate(service, request, expected_status=500, policy=policy)
|
|
64
|
+
assert result == response_data
|
|
65
|
+
|
|
66
|
+
assert send_mocker.call_args_list == [
|
|
67
|
+
call(method='GET', url='https://localhost:8000', cfg={'params': {'q1': 1, 'q2': '2'}})
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
async def test_http_service__retry_error(mocker):
|
|
72
|
+
send_mocker = mocker.patch('http_misc.services.HttpService._send')
|
|
73
|
+
send_mocker.side_effect = RetryError()
|
|
74
|
+
|
|
75
|
+
max_retry = 5
|
|
76
|
+
policy = AsyncRetryPolicy(max_retry=max_retry, backoff_factor=0.001, jitter=0.001)
|
|
77
|
+
service = HttpService()
|
|
78
|
+
request = {
|
|
79
|
+
'method': 'GET',
|
|
80
|
+
'url': 'https://localhost:8000',
|
|
81
|
+
'cfg': {
|
|
82
|
+
'params': {
|
|
83
|
+
'q1': 1,
|
|
84
|
+
'q2': '2'
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
with pytest.raises(MaxRetryError, match=f'Exceeded the maximum number of attempts {max_retry}.'):
|
|
89
|
+
await policy.apply(service.send_request, **request)
|
|
90
|
+
|
|
91
|
+
assert send_mocker.call_args_list == [
|
|
92
|
+
call(method='GET', url='https://localhost:8000', cfg={'params': {'q1': 1, 'q2': '2'}})
|
|
93
|
+
] * (max_retry + 1)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def test_http_service__error(mocker):
|
|
97
|
+
send_mocker = mocker.patch('http_misc.services.HttpService._send')
|
|
98
|
+
send_mocker.side_effect = Exception('Test')
|
|
99
|
+
|
|
100
|
+
max_retry = 5
|
|
101
|
+
policy = AsyncRetryPolicy(max_retry=max_retry, backoff_factor=0.001, jitter=0.001)
|
|
102
|
+
service = HttpService()
|
|
103
|
+
request = {
|
|
104
|
+
'method': 'GET',
|
|
105
|
+
'url': 'https://localhost:8000',
|
|
106
|
+
'cfg': {
|
|
107
|
+
'params': {
|
|
108
|
+
'q1': 1,
|
|
109
|
+
'q2': '2'
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
with pytest.raises(Exception, match=f'Test'):
|
|
114
|
+
await policy.apply(service.send_request, **request)
|
|
115
|
+
|
|
116
|
+
assert send_mocker.call_args_list == [
|
|
117
|
+
call(method='GET', url='https://localhost:8000', cfg={'params': {'q1': 1, 'q2': '2'}})
|
|
118
|
+
]
|
|
@@ -1,237 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import logging
|
|
3
|
-
import random
|
|
4
|
-
import uuid
|
|
5
|
-
from abc import ABC, abstractmethod
|
|
6
|
-
from contextlib import asynccontextmanager
|
|
7
|
-
from dataclasses import dataclass
|
|
8
|
-
from enum import Enum
|
|
9
|
-
|
|
10
|
-
from aiohttp import ContentTypeError, ClientSession
|
|
11
|
-
|
|
12
|
-
from http_misc import http_utils, errors
|
|
13
|
-
|
|
14
|
-
DEFAULT_RETRY_ON_STATUSES = frozenset([413, 429, 503, 504])
|
|
15
|
-
logger = logging.getLogger(__name__)
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
@dataclass
|
|
19
|
-
class ServiceResponse:
|
|
20
|
-
""" Ответ сервиса """
|
|
21
|
-
status: int
|
|
22
|
-
response_data: any = None
|
|
23
|
-
raw_response: any = None
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
class OnErrorResult(Enum):
|
|
27
|
-
"""
|
|
28
|
-
SILENT - проигнорировать ошибку
|
|
29
|
-
THROW - возбудить ошибку
|
|
30
|
-
REFRESH - при ошибке выполнить действие повторно
|
|
31
|
-
"""
|
|
32
|
-
SILENT = 1
|
|
33
|
-
THROW = 2
|
|
34
|
-
REFRESH = 3
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
@dataclass
|
|
38
|
-
class RetryPolicy:
|
|
39
|
-
max_retry: int | None = 10
|
|
40
|
-
retry_on_statuses: set[int] | None = DEFAULT_RETRY_ON_STATUSES
|
|
41
|
-
backoff_factor: int | None = 0.3
|
|
42
|
-
jitter: int | None = 0.1
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
class Transformer(ABC):
|
|
46
|
-
@abstractmethod
|
|
47
|
-
async def modify(self, request_id, *args, **kwargs):
|
|
48
|
-
""" Изменение параметров запроса или ответа """
|
|
49
|
-
return args, kwargs
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
class BaseService(ABC):
|
|
53
|
-
"""
|
|
54
|
-
Abstract service
|
|
55
|
-
"""
|
|
56
|
-
|
|
57
|
-
def __init__(self, retry_policy: RetryPolicy | None = None,
|
|
58
|
-
request_preproc: list[Transformer] | None = None,
|
|
59
|
-
response_preproc: list[Transformer] | None = None):
|
|
60
|
-
""" Сервис """
|
|
61
|
-
self.retry_policy = retry_policy or RetryPolicy()
|
|
62
|
-
self.request_preproc = request_preproc
|
|
63
|
-
self.response_preproc = response_preproc
|
|
64
|
-
self._request_count_manager = _RequestCountManager()
|
|
65
|
-
|
|
66
|
-
@abstractmethod
|
|
67
|
-
async def _send(self, request_id: uuid.UUID, *args, **kwargs) -> ServiceResponse:
|
|
68
|
-
"""
|
|
69
|
-
Abstract _send
|
|
70
|
-
"""
|
|
71
|
-
raise NotImplementedError('Not implemented _send method')
|
|
72
|
-
|
|
73
|
-
async def send_request(self, *args, **kwargs):
|
|
74
|
-
""" Вызов внешнего сервиса """
|
|
75
|
-
request_id = self._request_count_manager.add()
|
|
76
|
-
try:
|
|
77
|
-
return await self._send_request(request_id, *args, **kwargs)
|
|
78
|
-
finally:
|
|
79
|
-
# обнуляем current_step для запроса, он выполнен успешно
|
|
80
|
-
self._request_count_manager.pop(request_id)
|
|
81
|
-
|
|
82
|
-
def get_current_step(self, request_id: uuid.UUID):
|
|
83
|
-
""" Получение количества попыток запроса """
|
|
84
|
-
return self._request_count_manager.get(request_id)
|
|
85
|
-
|
|
86
|
-
async def _send_request(self, request_id: uuid.UUID, *args, **kwargs) -> ServiceResponse:
|
|
87
|
-
"""
|
|
88
|
-
Вызов внешнего сервиса
|
|
89
|
-
"""
|
|
90
|
-
try:
|
|
91
|
-
args, kwargs = await self._before_send(request_id, *args, **kwargs)
|
|
92
|
-
logger.debug('Send request %s: %s; %s', self.get_current_step(request_id), args, kwargs)
|
|
93
|
-
service_response = await self._send(request_id, *args, **kwargs)
|
|
94
|
-
service_response = await self._transform_response(request_id, service_response)
|
|
95
|
-
logger.debug('Response %s: %s, %s', self.get_current_step(request_id),
|
|
96
|
-
service_response.status, service_response.response_data)
|
|
97
|
-
|
|
98
|
-
if self.retry_policy.retry_on_statuses and service_response.status in self.retry_policy.retry_on_statuses:
|
|
99
|
-
raise errors.RetryError()
|
|
100
|
-
|
|
101
|
-
return service_response
|
|
102
|
-
except Exception as ex:
|
|
103
|
-
if isinstance(ex, errors.RetryError):
|
|
104
|
-
on_error_result = OnErrorResult.REFRESH
|
|
105
|
-
else:
|
|
106
|
-
on_error_result = await self._on_error(ex, *args, **kwargs)
|
|
107
|
-
|
|
108
|
-
if on_error_result == OnErrorResult.THROW:
|
|
109
|
-
logger.exception(ex)
|
|
110
|
-
raise ex
|
|
111
|
-
|
|
112
|
-
if on_error_result == OnErrorResult.SILENT:
|
|
113
|
-
logger.exception(ex)
|
|
114
|
-
return ServiceResponse(status=-1)
|
|
115
|
-
|
|
116
|
-
if on_error_result == OnErrorResult.REFRESH:
|
|
117
|
-
return await self._resend_request(request_id, ex, *args, **kwargs)
|
|
118
|
-
|
|
119
|
-
raise NotImplementedError(f'on_error_result "{on_error_result}" not implemented') from ex
|
|
120
|
-
|
|
121
|
-
async def _transform_response(self, request_id: uuid.UUID, response: ServiceResponse) -> ServiceResponse:
|
|
122
|
-
""" Преобразование ответа для возврата пользователю """
|
|
123
|
-
if self.response_preproc:
|
|
124
|
-
for response_preproc in self.response_preproc:
|
|
125
|
-
response = await response_preproc.modify(request_id, response)
|
|
126
|
-
return response
|
|
127
|
-
|
|
128
|
-
async def _before_send(self, request_id: uuid.UUID, *args, **kwargs):
|
|
129
|
-
""" Действие перед вызовом """
|
|
130
|
-
if self.request_preproc:
|
|
131
|
-
for request_preproc in self.request_preproc:
|
|
132
|
-
args, kwargs = await request_preproc.modify(request_id, *args, **kwargs)
|
|
133
|
-
return args, kwargs
|
|
134
|
-
|
|
135
|
-
async def _on_error(self, ex: Exception, *args, **kwargs) -> OnErrorResult: # pylint: disable=unused-argument
|
|
136
|
-
"""
|
|
137
|
-
Действие на возникновение ошибки.
|
|
138
|
-
SILENT - продолжить работу без возникновения ошибки
|
|
139
|
-
THROW - выкинуть исключение дальше
|
|
140
|
-
REFRESH - сделать повторный вызов сервиса
|
|
141
|
-
"""
|
|
142
|
-
return OnErrorResult.THROW
|
|
143
|
-
|
|
144
|
-
async def _resend_request(self, request_id: uuid.UUID, ex: Exception, *args, **kwargs):
|
|
145
|
-
""" Повторная отправка запроса """
|
|
146
|
-
self._request_count_manager.inc(request_id)
|
|
147
|
-
|
|
148
|
-
current_step = self.get_current_step(request_id)
|
|
149
|
-
logger.debug('Repeat request #%s.', current_step)
|
|
150
|
-
|
|
151
|
-
if current_step >= self.retry_policy.max_retry:
|
|
152
|
-
raise errors.MaxRetryError(f'Exceeded the maximum number of attempts {self.retry_policy.max_retry}') from ex
|
|
153
|
-
|
|
154
|
-
sleep_seconds = self.retry_policy.backoff_factor * (2 ** (current_step - 1))
|
|
155
|
-
sleep_seconds += random.normalvariate(0, sleep_seconds * self.retry_policy.jitter)
|
|
156
|
-
await asyncio.sleep(sleep_seconds)
|
|
157
|
-
|
|
158
|
-
return await self._send_request(request_id, *args, **kwargs)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
class HttpService(BaseService):
|
|
162
|
-
"""
|
|
163
|
-
Вызов по протоколу http
|
|
164
|
-
"""
|
|
165
|
-
|
|
166
|
-
def __init__(self, *args, client_session: ClientSession | None = None, **kwargs):
|
|
167
|
-
super().__init__(*args, **kwargs)
|
|
168
|
-
self.client_session = client_session
|
|
169
|
-
|
|
170
|
-
async def _send(self, request_id: uuid.UUID, *args, **kwargs) -> ServiceResponse:
|
|
171
|
-
method = kwargs.get('method', 'get')
|
|
172
|
-
url = kwargs.get('url', None)
|
|
173
|
-
if url is None:
|
|
174
|
-
raise ValueError('Url is none')
|
|
175
|
-
url = str(url)
|
|
176
|
-
|
|
177
|
-
cfg = kwargs.get('cfg', {})
|
|
178
|
-
if not isinstance(cfg, dict):
|
|
179
|
-
raise ValueError('Invalid cfg type. Must be dict.')
|
|
180
|
-
|
|
181
|
-
if url.lower().startswith('https://') and 'ssl' not in cfg:
|
|
182
|
-
cfg['ssl'] = False
|
|
183
|
-
|
|
184
|
-
async with self._use_client_session() as session:
|
|
185
|
-
async with session.request(method, url, **cfg) as response:
|
|
186
|
-
response_data = await _get_response_content(response)
|
|
187
|
-
return ServiceResponse(status=response.status, response_data=response_data, raw_response=response)
|
|
188
|
-
|
|
189
|
-
@asynccontextmanager
|
|
190
|
-
async def _use_client_session(self):
|
|
191
|
-
if self.client_session is not None:
|
|
192
|
-
yield self.client_session
|
|
193
|
-
else:
|
|
194
|
-
async with ClientSession(json_serialize=http_utils.json_dumps) as session:
|
|
195
|
-
yield session
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
class _RequestCountManager:
|
|
199
|
-
def __init__(self):
|
|
200
|
-
self._requests: dict[uuid.UUID, int] = {}
|
|
201
|
-
|
|
202
|
-
def add(self) -> uuid.UUID:
|
|
203
|
-
""" Инициализация запроса """
|
|
204
|
-
request_id = uuid.uuid4()
|
|
205
|
-
self._requests[request_id] = 0
|
|
206
|
-
return request_id
|
|
207
|
-
|
|
208
|
-
def exist(self, request_id: uuid.UUID) -> bool:
|
|
209
|
-
""" Проверка наличия запроса """
|
|
210
|
-
if request_id not in self._requests:
|
|
211
|
-
raise KeyError(f'Request {str(request_id)} not in registry')
|
|
212
|
-
|
|
213
|
-
return True
|
|
214
|
-
|
|
215
|
-
def pop(self, request_id: uuid.UUID) -> int | None:
|
|
216
|
-
""" Удаление запроса """
|
|
217
|
-
self.exist(request_id)
|
|
218
|
-
return self._requests.pop(request_id)
|
|
219
|
-
|
|
220
|
-
def get(self, request_id: uuid.UUID) -> int:
|
|
221
|
-
""" Получение количества попыток запроса """
|
|
222
|
-
self.exist(request_id)
|
|
223
|
-
return self._requests[request_id]
|
|
224
|
-
|
|
225
|
-
def inc(self, request_id: uuid.UUID) -> int:
|
|
226
|
-
""" Увеличение количества попыток на 1 """
|
|
227
|
-
self.exist(request_id)
|
|
228
|
-
self._requests[request_id] += 1
|
|
229
|
-
|
|
230
|
-
return self._requests[request_id]
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
async def _get_response_content(response):
|
|
234
|
-
try:
|
|
235
|
-
return await response.json()
|
|
236
|
-
except ContentTypeError:
|
|
237
|
-
return await response.text()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|